rewrite/testing: add additional test for live rewrite post, invalid post

htmlrewrite: annotate untestable sections (unimplemented, 2.6 only exceptions)

pywb/framework/wbrequestresponse.py

@@ -179,13 +179,6 @@ class WbResponse(object):
         return WbResponse(StatusAndHeaders(status, redir_headers))
 
     def __call__(self, env, start_response):
-
-        # PERF
-        perfstats = env.get('X_PERF')
-        if perfstats:
-            self.status_headers.headers.append(('X-Archive-Perf-Stats',
-                                                str(perfstats)))
-
         start_response(self.status_headers.statusline,
                        self.status_headers.headers)
 

pywb/rewrite/html_rewriter.py

@@ -101,12 +101,9 @@ class HTMLRewriterMixin(object):
         if not m:
             return meta_refresh
 
-        try:
+        meta_refresh = (meta_refresh[:m.start(1)] +
-            meta_refresh = (meta_refresh[:m.start(1)] +
+                        self._rewrite_url(m.group(1)) +
-                            self._rewrite_url(m.group(1)) +
+                        meta_refresh[m.end(1):])
-                            meta_refresh[m.end(1):])
-        except Exception:
-            pass
 
         return meta_refresh
     # ===========================
@@ -136,7 +133,7 @@ class HTMLRewriterMixin(object):
                 return value.lower() == attr_value.lower()
         return False
 
-    def _rewrite_tag_attrs(self, tag, tag_attrs, escape=False):
+    def _rewrite_tag_attrs(self, tag, tag_attrs):
         # special case: script or style parse context
         if ((tag in self.STATE_TAGS) and not self._wb_parse_context):
             self._wb_parse_context = tag
@@ -197,7 +194,7 @@ class HTMLRewriterMixin(object):
                                          rebase_rewriter(attr_value))
 
             # write the attr!
-            self._write_attr(attr_name, attr_value, escape=escape)
+            self._write_attr(attr_name, attr_value)
 
         return True
 
@@ -217,12 +214,10 @@ class HTMLRewriterMixin(object):
 
         return True
 
-    def _write_attr(self, name, value, escape=False):
+    def _write_attr(self, name, value):
         # parser doesn't differentiate between 'attr=""' and just 'attr'
         # 'attr=""' is more common, so use that form
         if value:
-            if escape:
-                value = cgi.escape(value, quote=True)
             self.out.write(' ' + name + '="' + value + '"')
         else:
             self.out.write(' ' + name + '=""')
@@ -259,8 +254,8 @@ class HTMLRewriterMixin(object):
 
         return result
 
-    def _internal_close(self):
+    def _internal_close(self):  # pragma: no cover
-        pass
+        raise NotImplementedError('Base method')
 
 
 #=================================================================
@@ -272,7 +267,8 @@ class HTMLRewriter(HTMLRewriterMixin, HTMLParser):
     def feed(self, string):
         try:
             HTMLParser.feed(self, string)
-        except HTMLParseError:
+        except HTMLParseError:  # pragma: no cover
+            # only raised in 2.6
             self.out.write(string)
 
     def _internal_close(self):
@@ -283,7 +279,8 @@ class HTMLRewriter(HTMLRewriterMixin, HTMLParser):
 
         try:
             HTMLParser.close(self)
-        except HTMLParseError:
+        except HTMLParseError:  # pragma: no cover
+            # only raised in 2.6
             pass
 
     # called to unescape attrs -- do not unescape!

pywb/rewrite/rewrite_live.py

@@ -130,14 +130,14 @@ class LiveRewriter(object):
 
         ts_err = url.split('///')
 
+        # fixup for accidental erroneous rewrite which has ///
+        # (unless file:///)
         if len(ts_err) > 1 and ts_err[0] != 'file:':
             url = 'http://' + ts_err[1]
 
         if url.startswith('//'):
             url = 'http:' + url
 
-        print 'URL ', url
-
         if is_http(url):
             (status_headers, stream) = self.fetch_http(url, env, req_headers,
                                                        follow_redirects,

pywb/utils/test/test_loaders.py

@@ -1,5 +1,5 @@
 #=================================================================
-"""
+r"""
 # LimitReader Tests
 >>> LimitReader(BytesIO('abcdefghjiklmnopqrstuvwxyz'), 10).read(26)
 'abcdefghji'
@@ -32,10 +32,14 @@ True
 >>> BlockLoader(HMACCookieMaker('test', 'test', 5)).load('http://example.com', 41, 14).read()
 'Example Domain'
 
-# fixed cookie
+# fixed cookie, range request
 >>> BlockLoader('some=value').load('http://example.com', 41, 14).read()
 'Example Domain'
 
+# range request
+>>> BlockLoader().load('http://example.com', 1262).read()
+'</html>\n'
+
 # test with extra id, ensure 4 parts of the A-B=C-D form are present
 >>> len(re.split('[-=]', HMACCookieMaker('test', 'test', 5).make('extra')))
 4

tests/test_integration.py

@@ -301,6 +301,11 @@ class TestWb:
         assert resp.status_int == 200
         assert '"data": "^"' in resp.body
 
+    def test_post_invalid(self):
+        # not json
+        resp = self.testapp.post_json('/pywb/20140610001255mp_/http://httpbin.org/post?foo=bar', {'data': '^'}, status=404)
+        assert resp.status_int == 404
+
     def test_post_redirect(self):
         # post handled without redirect (since 307 not allowed)
         resp = self.testapp.post('/post', {'foo': 'bar', 'test': 'abc'}, headers=[('Referer', 'http://localhost:8080/pywb/2014mp_/http://httpbin.org/post')])
@@ -308,7 +313,6 @@ class TestWb:
         assert '"foo": "bar"' in resp.body
         assert '"test": "abc"' in resp.body
 
-
     def test_excluded_content(self):
         resp = self.testapp.get('/pywb/mp_/http://www.iana.org/_img/bookmark_icon.ico', status = 403)
         assert resp.status_int == 403

tests/test_live_rewriter.py

@@ -17,6 +17,13 @@ class TestLiveRewriter:
         resp = self.testapp.get('/rewrite/mp_/http://facebook.com/')
         assert resp.status_int == 301
 
+    def test_live_rewrite_post(self):
+        resp = self.testapp.post('/rewrite/mp_/httpbin.org/post', {'foo': 'bar', 'test': 'abc'})
+        assert resp.status_int == 200
+        assert '"foo": "bar"' in resp.body
+        assert '"test": "abc"' in resp.body
+        assert resp.status_int == 200
+
     def test_live_rewrite_frame(self):
         resp = self.testapp.get('/rewrite/http://example.com/')
         assert resp.status_int == 200