From 41486f5f825f1b49a189927e40a090e6a7167f96 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 27 Mar 2018 12:51:37 -0700 Subject: [PATCH 1/2] logging tweaks --- setup.py | 2 +- warcprox/__init__.py | 4 ++-- warcprox/writerthread.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6a79e63..f47da6f 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev157', + version='2.4b2.dev158', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/__init__.py b/warcprox/__init__.py index 76abafa..76d733d 100644 --- a/warcprox/__init__.py +++ b/warcprox/__init__.py @@ -130,7 +130,7 @@ class BasePostfetchProcessor(threading.Thread): raise Exception('not implemented') def _run(self): - logging.info('%s starting up', self) + self.logger.info('%s starting up', self) self._startup() while not self.stop.is_set(): try: @@ -140,7 +140,7 @@ class BasePostfetchProcessor(threading.Thread): except queue.Empty: if self.stop.is_set(): break - logging.info('%s shutting down', self) + self.logger.info('%s shutting down', self) self._shutdown() except Exception as e: if isinstance(e, OSError) and e.errno == 28: diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index a3eeedb..927c628 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -44,6 +44,10 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1) self.batch = set() + def _startup(self): + self.logger.info('%s threads', self.pool._max_workers) + warcprox.BaseStandardPostfetchProcessor._startup(self) + def _get_process_put(self): try: recorded_url = self.inq.get(block=True, timeout=0.5) From 7f1c7f532eb948bc903024238a16e59e328bd3f7 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 28 Mar 2018 18:04:54 -0700 Subject: [PATCH 2/2] stop swallowing exception on _proxy_request() --- setup.py | 2 +- warcprox/mitmproxy.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f47da6f..629ed64 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev158', + version='2.4b2.dev159', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 2776a97..8778822 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -462,6 +462,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): self._conn_pool._put_conn(self._remote_server_conn) except: self._remote_server_conn.sock.close() + raise finally: if prox_rec_res: prox_rec_res.close()