backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Improve Connection Pool

Browse Source 
Set connection pool maxsize to 6 (borrowing from browser behavior).

Set num_pools to `max_threads / 6` but set a minimum of 200 for the cases
that we use a very low number of `max_threads`.

Remove `connection_is_fine` variable from connection code.

Fix http headers bug introduced in the previous commit.
This commit is contained in:
Vangelis Banos 2018-03-16 21:06:34 +00:00
parent 1d5692dd13
commit 0002d29f0d
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
warcprox/mitmproxy.py
View File

@ -176,7 +176,7 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
for k,v in self.msg.items(): for k,v in self.msg.items():
if k.lower() not in ( if k.lower() not in (
'connection', 'proxy-connection', 'connection', 'proxy-connection', 'keep-alive',
'proxy-authenticate', 'proxy-authorization', 'upgrade', 'proxy-authenticate', 'proxy-authorization', 'upgrade',
'strict-transport-security'): 'strict-transport-security'):
status_and_headers += '{}: {}\r\n'.format(k, v) status_and_headers += '{}: {}\r\n'.format(k, v)
@ -247,7 +247,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
''' '''
self._conn_pool = self.server.remote_connection_pool.connection_from_host( self._conn_pool = self.server.remote_connection_pool.connection_from_host(
host=self.hostname, port=int(self.port), scheme='http', host=self.hostname, port=int(self.port), scheme='http',
pool_kwargs={'maxsize': 30}) pool_kwargs={'maxsize': 6})
self._remote_server_conn = self._conn_pool._get_conn() self._remote_server_conn = self._conn_pool._get_conn()
if is_connection_dropped(self._remote_server_conn): if is_connection_dropped(self._remote_server_conn):
@ -426,7 +426,6 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
req += self.rfile.read(int(self.headers['Content-Length'])) req += self.rfile.read(int(self.headers['Content-Length']))
prox_rec_res = None prox_rec_res = None
connection_is_fine = False
try: try:
self.logger.debug('sending to remote server req=%r', req) self.logger.debug('sending to remote server req=%r', req)
@ -450,17 +449,16 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self._max_resource_size, self.url) self._max_resource_size, self.url)
break break
connection_is_fine = True
self.log_request(prox_rec_res.status, prox_rec_res.recorder.len) self.log_request(prox_rec_res.status, prox_rec_res.recorder.len)
finally:
# Let's close off the remote end. If remote connection is fine, # Let's close off the remote end. If remote connection is fine,
# put it back in the pool to reuse it later. # put it back in the pool to reuse it later.
if not is_connection_dropped(self._remote_server_conn):
self._conn_pool._put_conn(self._remote_server_conn)
except:
self._remote_server_conn.sock.close()
finally:
if prox_rec_res: if prox_rec_res:
prox_rec_res.close() prox_rec_res.close()
if connection_is_fine and not is_connection_dropped(self._remote_server_conn):
self._conn_pool._put_conn(self._remote_server_conn)
else:
self._remote_server_conn.sock.close()
return req, prox_rec_res return req, prox_rec_res

2
warcprox/warcproxy.py
View File

@ -389,7 +389,7 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
self.stats_db = stats_db self.stats_db = stats_db
self.options = options self.options = options
self.remote_connection_pool = PoolManager( self.remote_connection_pool = PoolManager(
num_pools=max(options.max_threads, 500) if options.max_threads else 500) num_pools=max(round(options.max_threads / 6), 200) if options.max_threads else 200)
server_address = ( server_address = (
options.address or 'localhost', options.address or 'localhost',
options.port if options.port is not None else 8000) options.port if options.port is not None else 8000)