backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'master' into qa

Browse Source 
* master:
  stop swallowing exception on _proxy_request()
  logging tweaks
This commit is contained in:
Noah Levitt 2018-03-28 18:05:13 -07:00
commit 623fb24bee
4 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev157', version='2.4b2.dev159',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
warcprox/__init__.py
View File

@ -130,7 +130,7 @@ class BasePostfetchProcessor(threading.Thread):
raise Exception('not implemented') raise Exception('not implemented')
def _run(self): def _run(self):
logging.info('%s starting up', self) self.logger.info('%s starting up', self)
self._startup() self._startup()
while not self.stop.is_set(): while not self.stop.is_set():
try: try:
@ -140,7 +140,7 @@ class BasePostfetchProcessor(threading.Thread):
except queue.Empty: except queue.Empty:
if self.stop.is_set(): if self.stop.is_set():
break break
logging.info('%s shutting down', self) self.logger.info('%s shutting down', self)
self._shutdown() self._shutdown()
except Exception as e: except Exception as e:
if isinstance(e, OSError) and e.errno == 28: if isinstance(e, OSError) and e.errno == 28:

1
warcprox/mitmproxy.py
View File

@ -462,6 +462,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self._conn_pool._put_conn(self._remote_server_conn) self._conn_pool._put_conn(self._remote_server_conn)
except: except:
self._remote_server_conn.sock.close() self._remote_server_conn.sock.close()
raise
finally: finally:
if prox_rec_res: if prox_rec_res:
prox_rec_res.close() prox_rec_res.close()

4
warcprox/writerthread.py
View File

@ -44,6 +44,10 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1) self.pool = futures.ThreadPoolExecutor(max_workers=options.writer_threads or 1)
self.batch = set() self.batch = set()
def _startup(self):
self.logger.info('%s threads', self.pool._max_workers)
warcprox.BaseStandardPostfetchProcessor._startup(self)
def _get_process_put(self): def _get_process_put(self):
try: try:
recorded_url = self.inq.get(block=True, timeout=0.5) recorded_url = self.inq.get(block=True, timeout=0.5)