From 32200db7ab546d610a00b28fe804334a49e6387f Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 13 Sep 2019 11:43:39 -0700 Subject: [PATCH] log long-running fetches --- warcprox/mitmproxy.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 6a7ce4c..9bac478 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -561,15 +561,18 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): 'bytes exceeded for URL %s', self._max_resource_size, self.url) break - elif (not 'content-length' in self.headers - and time.time() - start > 3 * 60 * 60): - prox_rec_res.truncated = b'time' - self._remote_server_conn.sock.shutdown(socket.SHUT_RDWR) - self._remote_server_conn.sock.close() - self.logger.info( - 'reached hard timeout of 3 hours fetching url ' - 'without content-length: %s', self.url) - break + elif time.time() - start > 3 * 60 * 60: + if not 'content-length' in self.headers: + prox_rec_res.truncated = b'time' + self._remote_server_conn.sock.shutdown(socket.SHUT_RDWR) + self._remote_server_conn.sock.close() + self.logger.info( + 'reached hard timeout of 3 hours fetching url ' + 'without content-length: %s', self.url) + break + else: + self.logger.info( + 'long-running fetch for URL %s', self.url) self.log_request(prox_rec_res.status, prox_rec_res.recorder.len) # Let's close off the remote end. If remote connection is fine,