followup on IncompleteRead

setup.py

@@ -42,7 +42,7 @@ except:
 
 setuptools.setup(
         name='warcprox',
-        version='2.4.4',
+        version='2.4.5',
         description='WARC writing MITM HTTP/S proxy',
         url='https://github.com/internetarchive/warcprox',
         author='Noah Levitt',

tests/test_warcprox.py

@@ -2243,6 +2243,11 @@ def test_incomplete_read(http_daemon, warcprox_, archiving_proxies):
         response = requests.get(
             url, proxies=archiving_proxies, verify=False, timeout=10)
 
+    # although `requests.get` raises exception here, other clients like
+    # browsers put up with the server misbehavior; warcprox does too, and will
+    # record the response verbatim in the warc; this `wait()` call tests
+    # that a warc record is written
+
     # wait for postfetch chain
     wait(lambda: warcprox_.proxy.running_stats.urls - urls_before == 1)
 

warcprox/mitmproxy.py

@@ -493,7 +493,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
                     buf = prox_rec_res.read(65536)
                 except http_client.IncompleteRead as e:
                     self.logger.warn('%s from %s', e, self.url)
-                    buf = b''
+                    buf = e.partial
 
                 if (self._max_resource_size and
                         prox_rec_res.recorder.len > self._max_resource_size):