backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

followup on IncompleteRead

Browse Source
This commit is contained in:
Noah Levitt 2019-04-15 00:17:50 -07:00
parent 5de2569430
commit f207e32f50
3 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -42,7 +42,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4.4', version='2.4.5',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

5
tests/test_warcprox.py
View File

@ -2243,6 +2243,11 @@ def test_incomplete_read(http_daemon, warcprox_, archiving_proxies):
response = requests.get( response = requests.get(
url, proxies=archiving_proxies, verify=False, timeout=10) url, proxies=archiving_proxies, verify=False, timeout=10)
# although `requests.get` raises exception here, other clients like
# browsers put up with the server misbehavior; warcprox does too, and will
# record the response verbatim in the warc; this `wait()` call tests
# that a warc record is written
# wait for postfetch chain # wait for postfetch chain
wait(lambda: warcprox_.proxy.running_stats.urls - urls_before == 1) wait(lambda: warcprox_.proxy.running_stats.urls - urls_before == 1)

2
warcprox/mitmproxy.py
View File

@ -493,7 +493,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
buf = prox_rec_res.read(65536) buf = prox_rec_res.read(65536)
except http_client.IncompleteRead as e: except http_client.IncompleteRead as e:
self.logger.warn('%s from %s', e, self.url) self.logger.warn('%s from %s', e, self.url)
buf = b'' buf = e.partial
if (self._max_resource_size and if (self._max_resource_size and
prox_rec_res.recorder.len > self._max_resource_size): prox_rec_res.recorder.len > self._max_resource_size):