backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

handle recoverable errors that happen while

Browse Source 
iterating over results!
This commit is contained in:
Noah Levitt 2018-09-17 12:03:51 -07:00
parent 95c4cff838
commit 968513cdb5
1 changed files with 19 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
doublethink/rethinker.py
View File

@ -48,14 +48,31 @@ class RethinkerWrapper(object):
result = self.wrapped.run(conn, db=db or self.rr.dbname) result = self.wrapped.run(conn, db=db or self.rr.dbname)
if hasattr(result, '__next__'): if hasattr(result, '__next__'):
is_iter = True is_iter = True
def gen(): def gen():
try: try:
yield # empty yield, see comment below yield # empty yield, see comment below
for x in result: while True:
yield x try:
x = next(result)
yield x
except StopIteration:
break
except r.ReqlOpFailedError as e:
if e.args and re.match(
'^Cannot perform.*replica.*',
e.args[0]):
self.logger.error(
'will keep trying after '
'potentially recoverable '
'error: %s', e)
time.sleep(0.5)
else:
raise
finally: finally:
result.close() result.close()
conn.close() conn.close()
g = gen() g = gen()
# Start executing the generator, leaving off after the # Start executing the generator, leaving off after the
# empty yield. If we didn't do this, and the caller never # empty yield. If we didn't do this, and the caller never