backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

override mitmproxy.PooledMixIn.get_request to put a cap on the number of open file handles

Browse Source
This commit is contained in:
Noah Levitt 2017-04-11 16:35:25 -07:00
parent cbefa37fd9
commit e9d6a8fcf4
2 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev66', version='2.1b1.dev67',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

15
warcprox/mitmproxy.py
View File

@ -55,6 +55,7 @@ except ImportError:
import resource import resource
import concurrent.futures import concurrent.futures
import urlcanon import urlcanon
import time
class ProxyingRecorder(object): class ProxyingRecorder(object):
""" """
@ -430,6 +431,20 @@ class PooledMixIn(socketserver.ThreadingMixIn):
def process_request(self, request, client_address): def process_request(self, request, client_address):
self.pool.submit(self.process_request_thread, request, client_address) self.pool.submit(self.process_request_thread, request, client_address)
def get_request(self):
'''
Waits until no other requests are waiting for a thread in the pool to
become available, then calls `socket.accept`.
This override is necessary for the size of the thread pool to act as a
cap on the number of open file handles.
'''
# neither threading.Condition Queue.not_empty nor Queue.not_full do
# what we need here, right?
while self.pool._work_queue.qsize() > 0:
time.sleep(0.5)
return self.socket.accept()
class MitmProxy(http_server.HTTPServer): class MitmProxy(http_server.HTTPServer):
def finish_request(self, request, client_address): def finish_request(self, request, client_address):
''' '''