backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

continue after unexpected error

Browse Source
This commit is contained in:
Noah Levitt 2015-10-30 01:15:03 +00:00
parent fd847f01cd
commit 3363b2ec95
1 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/writerthread.py
View File

@ -39,6 +39,7 @@ class WarcWriterThread(threading.Thread):
cProfile.runctx('self._run()', globals(), locals(), sort='cumulative')
def _run(self):
while not self.stop.is_set():
try:
self.setName('WarcWriterThread(tid={})'.format(warcprox.gettid()))
while True:
@ -63,7 +64,8 @@ class WarcWriterThread(threading.Thread):
self.logger.info('WarcWriterThread shutting down')
self.writer_pool.close_writers()
except:
self.logger.critical("WarcWriterThread shutting down after unexpected error", exc_info=True)
self.logger.critical("WarcWriterThread will try to continue after unexpected error", exc_info=True)
time.sleep(0.5)
# closest thing we have to heritrix crawl log at the moment
def _log(self, recorded_url, records):