From e9d6a8fcf4d8215104a86ce9fe13e1b13cd1a9d4 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 11 Apr 2017 16:35:25 -0700 Subject: [PATCH] override mitmproxy.PooledMixIn.get_request to put a cap on the number of open file handles --- setup.py | 2 +- warcprox/mitmproxy.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8ddc828..7b449d6 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ except: setuptools.setup( name='warcprox', - version='2.1b1.dev66', + version='2.1b1.dev67', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 470e952..e15ed3c 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -55,6 +55,7 @@ except ImportError: import resource import concurrent.futures import urlcanon +import time class ProxyingRecorder(object): """ @@ -430,6 +431,20 @@ class PooledMixIn(socketserver.ThreadingMixIn): def process_request(self, request, client_address): self.pool.submit(self.process_request_thread, request, client_address) + def get_request(self): + ''' + Waits until no other requests are waiting for a thread in the pool to + become available, then calls `socket.accept`. + + This override is necessary for the size of the thread pool to act as a + cap on the number of open file handles. + ''' + # neither threading.Condition Queue.not_empty nor Queue.not_full do + # what we need here, right? + while self.pool._work_queue.qsize() > 0: + time.sleep(0.5) + return self.socket.accept() + class MitmProxy(http_server.HTTPServer): def finish_request(self, request, client_address): '''