backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

logging tweaks

Browse Source
This commit is contained in:
Noah Levitt 2018-03-27 12:51:37 -07:00
parent c79b89108a
commit 41486f5f82
3 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev157', version='2.4b2.dev158',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
warcprox/__init__.py
View File

@ -130,7 +130,7 @@ class BasePostfetchProcessor(threading.Thread):
raise Exception('not implemented') raise Exception('not implemented')
def _run(self): def _run(self):
logging.info('%s starting up', self) self.logger.info('%s starting up', self)
self._startup() self._startup()
while not self.stop.is_set(): while not self.stop.is_set():
try: try:
@ -140,7 +140,7 @@ class BasePostfetchProcessor(threading.Thread):
except queue.Empty: except queue.Empty:
if self.stop.is_set(): if self.stop.is_set():
break break
logging.info('%s shutting down', self) self.logger.info('%s shutting down', self)
self._shutdown() self._shutdown()
except Exception as e: except Exception as e:
if isinstance(e, OSError) and e.errno == 28: if isinstance(e, OSError) and e.errno == 28:

4
warcprox/writerthread.py
View File

@ -44,6 +44,10 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1) self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1)
self.batch = set() self.batch = set()
def _startup(self):
self.logger.info('%s threads', self.pool._max_workers)
warcprox.BaseStandardPostfetchProcessor._startup(self)
def _get_process_put(self): def _get_process_put(self):
try: try:
recorded_url = self.inq.get(block=True, timeout=0.5) recorded_url = self.inq.get(block=True, timeout=0.5)