From 84a46e4323265b40950d7a57b94db5ca5e7d4b34 Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Fri, 20 Sep 2019 05:55:51 +0000 Subject: [PATCH 1/2] Increase remote_connection_pool maxsize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We noticed a lot of log entries like this in production: ``` WARNING:urllib3.connectionpool:Connection pool is full, discarding connection: static.xx.fbcdn.net ``` this happens because we use a `PoolManager` and create a number of pools (param `num_pools`) but the number of connections each pool can have is just 1 by default (param `maxsize` is 1 by default). `urllib3` docs say: `maxsize` – Number of connections to save that can be reused. More than 1 is useful in multithreaded situations. Ref: https://urllib3.readthedocs.io/en/1.2.1/pools.html#urllib3.connectionpool.HTTPConnectionPool I suggest to use `maxsize=10` and re-evaluate after some time if its big enough. This improvement will boost performance as we'll reuse more connections to remote hosts. --- warcprox/mitmproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 2d7f9b6..929bc49 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -777,7 +777,7 @@ class SingleThreadedMitmProxy(http_server.HTTPServer): self.bad_hostnames_ports_lock = RLock() self.remote_connection_pool = PoolManager( - num_pools=max((options.max_threads or 0) // 6, 400)) + num_pools=max((options.max_threads or 0) // 6, 400), maxsize=10) if options.onion_tor_socks_proxy: try: From 407e89025844cfd8261aa43e4fdbea33e244c801 Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Sat, 21 Sep 2019 09:29:19 +0000 Subject: [PATCH 2/2] Set connection pool maxsize=6 --- warcprox/mitmproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 929bc49..6b32a40 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -777,7 +777,7 @@ class SingleThreadedMitmProxy(http_server.HTTPServer): self.bad_hostnames_ports_lock = RLock() self.remote_connection_pool = PoolManager( - num_pools=max((options.max_threads or 0) // 6, 400), maxsize=10) + num_pools=max((options.max_threads or 0) // 6, 400), maxsize=6) if options.onion_tor_socks_proxy: try: