From 1d5692dd13f0a6ecbcc35adae7d4bab3f2798330 Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Fri, 16 Mar 2018 13:10:29 +0000 Subject: [PATCH] Reduce the PoolManager num_pools size and fix bugs Define PoolManager num_pools size as `max(max_threads, 500)` and reduce each pool size from 100 to 30. The aim is to limit the total number of open connections. Fix remote SOCKS connection typo. Now that we reuse remote connections, its better NOT to remove the `keep-alive` request header. We need to send it to the remote host to make it keep the connection open if possible. --- warcprox/mitmproxy.py | 6 +++--- warcprox/warcproxy.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 9d076e3..1482210 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -176,7 +176,7 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse): for k,v in self.msg.items(): if k.lower() not in ( - 'connection', 'proxy-connection', 'keep-alive', + 'connection', 'proxy-connection', 'proxy-authenticate', 'proxy-authorization', 'upgrade', 'strict-transport-security'): status_and_headers += '{}: {}\r\n'.format(k, v) @@ -247,7 +247,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): ''' self._conn_pool = self.server.remote_connection_pool.connection_from_host( host=self.hostname, port=int(self.port), scheme='http', - pool_kwargs={'maxsize': 100}) + pool_kwargs={'maxsize': 30}) self._remote_server_conn = self._conn_pool._get_conn() if is_connection_dropped(self._remote_server_conn): @@ -257,7 +257,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): self.onion_tor_socks_proxy_host, self.onion_tor_socks_proxy_port or 1080, self.hostname) self._remote_server_conn.sock = socks.socksocket() - self._remote_server_sock.set_proxy( + self._remote_server_conn.sock.set_proxy( socks.SOCKS5, addr=self.onion_tor_socks_proxy_host, port=self.onion_tor_socks_proxy_port, rdns=True) else: diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py index c66c33d..88a9c34 100644 --- a/warcprox/warcproxy.py +++ b/warcprox/warcproxy.py @@ -388,7 +388,8 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object): self.status_callback = status_callback self.stats_db = stats_db self.options = options - self.remote_connection_pool = PoolManager(num_pools=2000) + self.remote_connection_pool = PoolManager( + num_pools=max(options.max_threads, 500) if options.max_threads else 500) server_address = ( options.address or 'localhost', options.port if options.port is not None else 8000)