backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

call stop() at shutdown if present on plugins

Browse Source
This commit is contained in:
Noah Levitt 2017-06-28 16:40:20 -07:00
parent 9ea3540d63
commit ad3e6f405d
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
warcprox/writerthread.py
View File

@ -106,14 +106,14 @@ class WarcWriterThread(threading.Thread):
self.idle = time.time() self.idle = time.time()
self.logger.info('WarcWriterThread shutting down') self.logger.info('WarcWriterThread shutting down')
self.writer_pool.close_writers() self._shutdown()
except Exception as e: except Exception as e:
if isinstance(e, OSError) and e.errno == 28: if isinstance(e, OSError) and e.errno == 28:
# OSError: [Errno 28] No space left on device # OSError: [Errno 28] No space left on device
self.logger.critical( self.logger.critical(
'shutting down due to fatal problem: %s: %s', 'shutting down due to fatal problem: %s: %s',
e.__class__.__name__, e) e.__class__.__name__, e)
self.writer_pool.close_writers() self._shutdown()
sys.exit(1) sys.exit(1)
self.logger.critical( self.logger.critical(
@ -121,6 +121,16 @@ class WarcWriterThread(threading.Thread):
'error', exc_info=True) 'error', exc_info=True)
time.sleep(0.5) time.sleep(0.5)
def _shutdown(self):
self.writer_pool.close_writers()
for listener in self.listeners:
if hasattr(listener, 'stop'):
try:
listener.stop()
except:
self.logger.error(
'%s raised exception', listener.stop, exc_info=True)
# closest thing we have to heritrix crawl log at the moment # closest thing we have to heritrix crawl log at the moment
def _log(self, recorded_url, records): def _log(self, recorded_url, records):
try: try: