http block loader: raise exception for 4xx, 5xx responses

tests: add tests for limitreader posting, fix charset for frame test

pywb/utils/loaders.py

@@ -392,6 +392,7 @@ class HttpLoader(BaseLoader):
             self.session = requests.Session()
 
         r = self.session.get(url, headers=headers, stream=True)
+        r.raise_for_status()
         return r.raw
 
 

pywb/utils/test/test_loaders.py

@@ -138,6 +138,7 @@ import pytest
 import six
 from six import StringIO
 from io import BytesIO
+import requests
 
 from pywb.utils.loaders import BlockLoader, HMACCookieMaker, to_file_url
 from pywb.utils.loaders import LimitReader, extract_client_cookie, extract_post_query
@@ -176,6 +177,14 @@ def test_s3_read_1():
     assert reader.readline() == b'WARC/1.0\r\n'
     assert reader.readline() == b'WARC-Type: response\r\n'
 
+def test_limit_post():
+    reader = LimitReader(BytesIO(b'abcdefg'), 3)
+    r = requests.request(method='POST',
+                         url='http://httpbin.org/post',
+                         data=reader,
+                         headers={'Content-Length': '3'})
+
+    assert '"abc"' in r.text
 
 # Error
 def test_err_no_such_file():

tests/test_live_rewriter.py

@@ -4,7 +4,6 @@ from pywb.framework.wsgi_wrappers import init_app
 import webtest
 import pywb.rewrite.rewrite_live
 
-
 #=================================================================
 class MockYTDWrapper(object):
     def extract_info(self, url):
@@ -47,6 +46,7 @@ class TestLiveRewriter:
     def test_live_live_frame(self):
         resp = self.testapp.get('/live/http://example.com/')
         assert resp.status_int == 200
+        resp.charset = 'utf-8'
         assert '<iframe ' in resp.text
         assert 'src="http://localhost:80/live/mp_/http://example.com/"' in resp.text, resp.text