Improve Connection Pool

Set connection pool maxsize to 6 (borrowing from browser behavior).

Set num_pools to `max_threads / 6` but set a minimum of 200 for the cases
that we use a very low number of `max_threads`.

Remove `connection_is_fine` variable from connection code.

Fix http headers bug introduced in the previous commit.
This commit is contained in:
Vangelis Banos 2018-03-16 21:06:34 +00:00
parent 1d5692dd13
commit 0002d29f0d
2 changed files with 8 additions and 10 deletions

View File

@ -176,7 +176,7 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
for k,v in self.msg.items():
if k.lower() not in (
'connection', 'proxy-connection',
'connection', 'proxy-connection', 'keep-alive',
'proxy-authenticate', 'proxy-authorization', 'upgrade',
'strict-transport-security'):
status_and_headers += '{}: {}\r\n'.format(k, v)
@ -247,7 +247,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
'''
self._conn_pool = self.server.remote_connection_pool.connection_from_host(
host=self.hostname, port=int(self.port), scheme='http',
pool_kwargs={'maxsize': 30})
pool_kwargs={'maxsize': 6})
self._remote_server_conn = self._conn_pool._get_conn()
if is_connection_dropped(self._remote_server_conn):
@ -426,7 +426,6 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
req += self.rfile.read(int(self.headers['Content-Length']))
prox_rec_res = None
connection_is_fine = False
try:
self.logger.debug('sending to remote server req=%r', req)
@ -450,17 +449,16 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self._max_resource_size, self.url)
break
connection_is_fine = True
self.log_request(prox_rec_res.status, prox_rec_res.recorder.len)
finally:
# Let's close off the remote end. If remote connection is fine,
# put it back in the pool to reuse it later.
if not is_connection_dropped(self._remote_server_conn):
self._conn_pool._put_conn(self._remote_server_conn)
except:
self._remote_server_conn.sock.close()
finally:
if prox_rec_res:
prox_rec_res.close()
if connection_is_fine and not is_connection_dropped(self._remote_server_conn):
self._conn_pool._put_conn(self._remote_server_conn)
else:
self._remote_server_conn.sock.close()
return req, prox_rec_res

View File

@ -389,7 +389,7 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
self.stats_db = stats_db
self.options = options
self.remote_connection_pool = PoolManager(
num_pools=max(options.max_threads, 500) if options.max_threads else 500)
num_pools=max(round(options.max_threads / 6), 200) if options.max_threads else 200)
server_address = (
options.address or 'localhost',
options.port if options.port is not None else 8000)