backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

log long-running fetches

Browse Source
This commit is contained in:
Barbara Miller 2019-09-13 11:43:39 -07:00
parent 1aa6b0c5d6
commit 32200db7ab
1 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
warcprox/mitmproxy.py
View File

@ -561,15 +561,18 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
'bytes exceeded for URL %s', 'bytes exceeded for URL %s',
self._max_resource_size, self.url) self._max_resource_size, self.url)
break break
elif (not 'content-length' in self.headers elif time.time() - start > 3 * 60 * 60:
and time.time() - start > 3 * 60 * 60): if not 'content-length' in self.headers:
prox_rec_res.truncated = b'time' prox_rec_res.truncated = b'time'
self._remote_server_conn.sock.shutdown(socket.SHUT_RDWR) self._remote_server_conn.sock.shutdown(socket.SHUT_RDWR)
self._remote_server_conn.sock.close() self._remote_server_conn.sock.close()
self.logger.info( self.logger.info(
'reached hard timeout of 3 hours fetching url ' 'reached hard timeout of 3 hours fetching url '
'without content-length: %s', self.url) 'without content-length: %s', self.url)
break break
else:
self.logger.info(
'long-running fetch for URL %s', self.url)
self.log_request(prox_rec_res.status, prox_rec_res.recorder.len) self.log_request(prox_rec_res.status, prox_rec_res.recorder.len)
# Let's close off the remote end. If remote connection is fine, # Let's close off the remote end. If remote connection is fine,