backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge pull request #151 from vbanos/fix-runtime-error

Browse Source 
Fix runtime error
This commit is contained in:
Barbara Miller 2020-07-09 15:12:00 -07:00 committed by GitHub
commit ca3d5d4edd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -34,7 +34,7 @@ before_install:
- ping -c2 trough - ping -c2 trough
install: install:
- pip install . pytest requests warcio mock - pip install .[trough] pytest requests warcio mock
before_script: before_script:
- docker exec trough bash -c 'while ! test -e /tmp/trough-read.out ; do sleep 0.5 ; done' || true - docker exec trough bash -c 'while ! test -e /tmp/trough-read.out ; do sleep 0.5 ; done' || true

2
setup.py
View File

@ -28,7 +28,7 @@ deps = [
'warctools>=4.10.0', 'warctools>=4.10.0',
'urlcanon>=0.3.0', 'urlcanon>=0.3.0',
'doublethink>=0.2.0.dev87', 'doublethink>=0.2.0.dev87',
'urllib3>=1.14', 'urllib3>=1.23',
'requests>=2.0.1', 'requests>=2.0.1',
'PySocks>=1.6.8', 'PySocks>=1.6.8',
'cryptography>=2.3', 'cryptography>=2.3',

18
warcprox/mitmproxy.py
View File

@ -649,13 +649,17 @@ class PooledMixIn(socketserver.ThreadingMixIn):
def process_request(self, request, client_address): def process_request(self, request, client_address):
self.active_requests[request] = doublethink.utcnow() self.active_requests[request] = doublethink.utcnow()
future = self.pool.submit( try:
self.process_request_thread, request, client_address) future = self.pool.submit(
future.add_done_callback( self.process_request_thread, request, client_address)
lambda f: self.active_requests.pop(request, None)) future.add_done_callback(
if future.done(): lambda f: self.active_requests.pop(request, None))
# avoid theoretical timing issue, in case process_request_thread if future.done():
# managed to finish before future.add_done_callback() ran # avoid theoretical timing issue, in case process_request_thread
# managed to finish before future.add_done_callback() ran
self.active_requests.pop(request, None)
except RuntimeError as exc:
self.logger.error("Error processing request %s", str(exc))
self.active_requests.pop(request, None) self.active_requests.pop(request, None)
def get_request(self): def get_request(self):