backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Reduce the PoolManager num_pools size and fix bugs

Browse Source 
Define PoolManager num_pools size as `max(max_threads, 500)` and reduce
each pool size from 100 to 30. The aim is to limit the total number of
open connections.

Fix remote SOCKS connection typo.

Now that we reuse remote connections, its better NOT to remove the
`keep-alive` request header. We need to send it to the remote host to make it
keep the connection open if possible.
This commit is contained in:
Vangelis Banos 2018-03-16 13:10:29 +00:00
parent 435b0ec24b
commit 1d5692dd13
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
warcprox/mitmproxy.py
View File

@ -176,7 +176,7 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
for k,v in self.msg.items():
if k.lower() not in (
'connection', 'proxy-connection', 'keep-alive',
'connection', 'proxy-connection',
'proxy-authenticate', 'proxy-authorization', 'upgrade',
'strict-transport-security'):
status_and_headers += '{}: {}\r\n'.format(k, v)
@ -247,7 +247,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
'''
self._conn_pool = self.server.remote_connection_pool.connection_from_host(
host=self.hostname, port=int(self.port), scheme='http',
pool_kwargs={'maxsize': 100})
pool_kwargs={'maxsize': 30})
self._remote_server_conn = self._conn_pool._get_conn()
if is_connection_dropped(self._remote_server_conn):
@ -257,7 +257,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self.onion_tor_socks_proxy_host,
self.onion_tor_socks_proxy_port or 1080, self.hostname)
self._remote_server_conn.sock = socks.socksocket()
self._remote_server_sock.set_proxy(
self._remote_server_conn.sock.set_proxy(
socks.SOCKS5, addr=self.onion_tor_socks_proxy_host,
port=self.onion_tor_socks_proxy_port, rdns=True)
else:

3
warcprox/warcproxy.py
View File

@ -388,7 +388,8 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
self.status_callback = status_callback
self.stats_db = stats_db
self.options = options
self.remote_connection_pool = PoolManager(num_pools=2000)
self.remote_connection_pool = PoolManager(
num_pools=max(options.max_threads, 500) if options.max_threads else 500)
server_address = (
options.address or 'localhost',
options.port if options.port is not None else 8000)