diff --git a/setup.py b/setup.py index 6a79e63..629ed64 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev157', + version='2.4b2.dev159', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/__init__.py b/warcprox/__init__.py index 76abafa..76d733d 100644 --- a/warcprox/__init__.py +++ b/warcprox/__init__.py @@ -130,7 +130,7 @@ class BasePostfetchProcessor(threading.Thread): raise Exception('not implemented') def _run(self): - logging.info('%s starting up', self) + self.logger.info('%s starting up', self) self._startup() while not self.stop.is_set(): try: @@ -140,7 +140,7 @@ class BasePostfetchProcessor(threading.Thread): except queue.Empty: if self.stop.is_set(): break - logging.info('%s shutting down', self) + self.logger.info('%s shutting down', self) self._shutdown() except Exception as e: if isinstance(e, OSError) and e.errno == 28: diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 2776a97..8778822 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -462,6 +462,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): self._conn_pool._put_conn(self._remote_server_conn) except: self._remote_server_conn.sock.close() + raise finally: if prox_rec_res: prox_rec_res.close() diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index a3eeedb..927c628 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -44,6 +44,10 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1) self.batch = set() + def _startup(self): + self.logger.info('%s threads', self.pool._max_workers) + warcprox.BaseStandardPostfetchProcessor._startup(self) + def _get_process_put(self): try: recorded_url = self.inq.get(block=True, timeout=0.5)