diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index 26d3d3c..0e73b97 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -106,14 +106,14 @@ class WarcWriterThread(threading.Thread): self.idle = time.time() self.logger.info('WarcWriterThread shutting down') - self.writer_pool.close_writers() + self._shutdown() except Exception as e: if isinstance(e, OSError) and e.errno == 28: # OSError: [Errno 28] No space left on device self.logger.critical( 'shutting down due to fatal problem: %s: %s', e.__class__.__name__, e) - self.writer_pool.close_writers() + self._shutdown() sys.exit(1) self.logger.critical( @@ -121,6 +121,16 @@ class WarcWriterThread(threading.Thread): 'error', exc_info=True) time.sleep(0.5) + def _shutdown(self): + self.writer_pool.close_writers() + for listener in self.listeners: + if hasattr(listener, 'stop'): + try: + listener.stop() + except: + self.logger.error( + '%s raised exception', listener.stop, exc_info=True) + # closest thing we have to heritrix crawl log at the moment def _log(self, recorded_url, records): try: