mirror of
https://github.com/internetarchive/warcprox.git
synced 2025-01-18 13:22:09 +01:00
Reduce the PoolManager num_pools size and fix bugs
Define PoolManager num_pools size as `max(max_threads, 500)` and reduce each pool size from 100 to 30. The aim is to limit the total number of open connections. Fix remote SOCKS connection typo. Now that we reuse remote connections, its better NOT to remove the `keep-alive` request header. We need to send it to the remote host to make it keep the connection open if possible.
This commit is contained in:
parent
435b0ec24b
commit
1d5692dd13
@ -176,7 +176,7 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
|
||||
|
||||
for k,v in self.msg.items():
|
||||
if k.lower() not in (
|
||||
'connection', 'proxy-connection', 'keep-alive',
|
||||
'connection', 'proxy-connection',
|
||||
'proxy-authenticate', 'proxy-authorization', 'upgrade',
|
||||
'strict-transport-security'):
|
||||
status_and_headers += '{}: {}\r\n'.format(k, v)
|
||||
@ -247,7 +247,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
|
||||
'''
|
||||
self._conn_pool = self.server.remote_connection_pool.connection_from_host(
|
||||
host=self.hostname, port=int(self.port), scheme='http',
|
||||
pool_kwargs={'maxsize': 100})
|
||||
pool_kwargs={'maxsize': 30})
|
||||
|
||||
self._remote_server_conn = self._conn_pool._get_conn()
|
||||
if is_connection_dropped(self._remote_server_conn):
|
||||
@ -257,7 +257,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
|
||||
self.onion_tor_socks_proxy_host,
|
||||
self.onion_tor_socks_proxy_port or 1080, self.hostname)
|
||||
self._remote_server_conn.sock = socks.socksocket()
|
||||
self._remote_server_sock.set_proxy(
|
||||
self._remote_server_conn.sock.set_proxy(
|
||||
socks.SOCKS5, addr=self.onion_tor_socks_proxy_host,
|
||||
port=self.onion_tor_socks_proxy_port, rdns=True)
|
||||
else:
|
||||
|
@ -388,7 +388,8 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
|
||||
self.status_callback = status_callback
|
||||
self.stats_db = stats_db
|
||||
self.options = options
|
||||
self.remote_connection_pool = PoolManager(num_pools=2000)
|
||||
self.remote_connection_pool = PoolManager(
|
||||
num_pools=max(options.max_threads, 500) if options.max_threads else 500)
|
||||
server_address = (
|
||||
options.address or 'localhost',
|
||||
options.port if options.port is not None else 8000)
|
||||
|
Loading…
x
Reference in New Issue
Block a user