diff --git a/.travis.yml b/.travis.yml index 4643ef3..d0f8030 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ before_install: - ping -c2 trough install: -- pip install . pytest requests warcio mock +- pip install .[trough] pytest requests warcio mock before_script: - docker exec trough bash -c 'while ! test -e /tmp/trough-read.out ; do sleep 0.5 ; done' || true diff --git a/setup.py b/setup.py index 96089bb..c8306b8 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ deps = [ 'warctools>=4.10.0', 'urlcanon>=0.3.0', 'doublethink>=0.2.0.dev87', - 'urllib3>=1.14', + 'urllib3>=1.23', 'requests>=2.0.1', 'PySocks>=1.6.8', 'cryptography>=2.3', diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 78d0ec3..4b0138e 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -649,13 +649,17 @@ class PooledMixIn(socketserver.ThreadingMixIn): def process_request(self, request, client_address): self.active_requests[request] = doublethink.utcnow() - future = self.pool.submit( - self.process_request_thread, request, client_address) - future.add_done_callback( - lambda f: self.active_requests.pop(request, None)) - if future.done(): - # avoid theoretical timing issue, in case process_request_thread - # managed to finish before future.add_done_callback() ran + try: + future = self.pool.submit( + self.process_request_thread, request, client_address) + future.add_done_callback( + lambda f: self.active_requests.pop(request, None)) + if future.done(): + # avoid theoretical timing issue, in case process_request_thread + # managed to finish before future.add_done_callback() ran + self.active_requests.pop(request, None) + except RuntimeError as exc: + self.logger.error("Error processing request %s", str(exc)) self.active_requests.pop(request, None) def get_request(self):