backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

shut down immediately on disk full error

Browse Source
This commit is contained in:
Noah Levitt 2017-03-28 12:39:41 -07:00
parent 73d934d0a4
commit 1c035153de
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev61', version='2.1b1.dev62',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

15
warcprox/writerthread.py
View File

@ -39,6 +39,7 @@ import hanzo.httptools
from hanzo import warctools from hanzo import warctools
import warcprox import warcprox
import cProfile import cProfile
import sys
class WarcWriterThread(threading.Thread): class WarcWriterThread(threading.Thread):
logger = logging.getLogger("warcprox.warcproxwriter.WarcWriterThread") logger = logging.getLogger("warcprox.warcproxwriter.WarcWriterThread")
@ -102,8 +103,18 @@ class WarcWriterThread(threading.Thread):
self.logger.info('WarcWriterThread shutting down') self.logger.info('WarcWriterThread shutting down')
self.writer_pool.close_writers() self.writer_pool.close_writers()
except: except BaseException as e:
self.logger.critical("WarcWriterThread will try to continue after unexpected error", exc_info=True) if isinstance(e, OSError) and e.errno == 28:
# OSError: [Errno 28] No space left on device
self.logger.critical(
'shutting down due to fatal problem: %s: %s',
e.__class__.__name__, e)
self.writer_pool.close_writers()
sys.exit(1)
self.logger.critical(
'WarcWriterThread will try to continue after unexpected '
'error', exc_info=True)
time.sleep(0.5) time.sleep(0.5)
# closest thing we have to heritrix crawl log at the moment # closest thing we have to heritrix crawl log at the moment