From 8ac0420cb2dc1c5e475a811919d26a94bbdd569b Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 4 Apr 2018 15:28:36 -0700 Subject: [PATCH] enable keepalive on test http server As of fairly recently, warcprox does keepalive with the remote server using the urllib3 connection pool. The test http server in test_warcprox.py was acting as if it supported keepalive (sending HTTP/1.1 and not sending "Connection: close"). But in fact it did not support keepalive. It was closing the connection after each request. Depending on the timing of what was happening in different threads, sometimes the client thread would try to send another request on a connection it still believed to be open for keepalive. Then the server side would complete its request processing and close the connection. This resulted in test failures with error messages like this (depending on python version): 2018-04-03 21:20:06,555 12586 ERROR MainThread warcprox.mitmproxy.MitmProxyHandler.do_COMMAND(mitmproxy.py:389) error from remote server(?) None: BadStatusLine("''",) 2018-04-04 19:06:29,599 11632 ERROR MainThread warcprox.mitmproxy.MitmProxyHandler.do_COMMAND(mitmproxy.py:389) error from remote server(?) None: RemoteDisconnected('Remote end closed connection without response',) For instance https://travis-ci.org/internetarchive/warcprox/jobs/362288603 --- tests/test_warcprox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_warcprox.py b/tests/test_warcprox.py index 26e1017..d175309 100755 --- a/tests/test_warcprox.py +++ b/tests/test_warcprox.py @@ -166,6 +166,9 @@ def chunkify(buf, chunk_size=13): # return outbuf.getvalue() class _TestHttpRequestHandler(http_server.BaseHTTPRequestHandler): + # enable keepalive + protocol_version = 'HTTP/1.1' + def build_response(self): m = re.match(r'^/([^/]+)/([^/]+)$', self.path) if m is not None: @@ -346,7 +349,6 @@ def https_daemon(request, cert): # http://www.piware.de/2011/01/creating-an-https-server-in-python/ https_daemon = http_server.HTTPServer(('localhost', 0), RequestHandlerClass=_TestHttpRequestHandler) - # https_daemon.socket = ssl.wrap_socket(httpd.socket, certfile='path/to/localhost.pem', server_side=True) https_daemon.socket = ssl.wrap_socket(https_daemon.socket, certfile=cert, server_side=True) logging.info('starting https://{}:{}'.format(https_daemon.server_address[0], https_daemon.server_address[1])) https_daemon_thread = threading.Thread(name='HttpsDaemonThread',