Local httpbin tests + LiveIndexSource improvement (#318)

tests and LiveIndexSource improvements:
- run local instance of httpbin in separate gevent server for any httpbin.org requests
- LiveIndexSource: has overridable get_load_url(), also use 'load_url' for HEAD check, remove unused proxy_url
- test update: add HttpBinLiveTests which patches LiveIndexSource.get_load_url() to redirect httpbin requests to local instance
- test update: just use httpbin.org/get instead of httpbin.org/anything, unsupported in older version (0.5.0) require for windows support
- setup: add 'httpbin==0.5.0' to test requires, remove jinja2 pin to old version

pywb/recorder/test/test_recorder.py

@@ -2,7 +2,7 @@ from gevent import monkey; monkey.patch_all()
 import gevent
 
 from pywb.warcserver.test.testutils import TempDirTests, LiveServerTests, BaseTestClass, to_path
-from pywb.warcserver.test.testutils import FakeRedisTests
+from pywb.warcserver.test.testutils import FakeRedisTests, HttpBinLiveTests
 
 import os
 import webtest
@@ -45,7 +45,7 @@ Cookie: boo=far\r\n\
 
 
 
-class TestRecorder(LiveServerTests, FakeRedisTests, TempDirTests, BaseTestClass):
+class TestRecorder(LiveServerTests, HttpBinLiveTests, FakeRedisTests, TempDirTests, BaseTestClass):
     @classmethod
     def setup_class(cls):
         super(TestRecorder, cls).setup_class()

pywb/warcserver/index/indexsource.py

@@ -196,8 +196,7 @@ class RemoteIndexSource(BaseIndexSource):
 
 #=============================================================================
 class LiveIndexSource(BaseIndexSource):
-    def __init__(self, proxy_url='{url}'):
+    def __init__(self):
-        self.proxy_url = proxy_url
         self._init_sesh(DefaultAdapters.live_adapter)
 
     def load_index(self, params):
@@ -209,14 +208,14 @@ class LiveIndexSource(BaseIndexSource):
         cdx['urlkey'] = params.get('key').decode('utf-8')
         cdx['timestamp'] = timestamp_now()
         cdx['url'] = params['url']
-        cdx['load_url'] = res_template(self.proxy_url, params)
+        cdx['load_url'] = self.get_load_url(params)
         cdx['is_live'] = 'true'
 
         mime = params.get('content_type', '')
 
         if params.get('filter') and not mime:
             try:
-                res = self.sesh.head(cdx['url'])
+                res = self.sesh.head(cdx['load_url'])
                 if res.status_code != 405:
                     cdx['status'] = str(res.status_code)
 
@@ -231,6 +230,9 @@ class LiveIndexSource(BaseIndexSource):
 
         return iter([cdx])
 
+    def get_load_url(self, params):
+        return params['url']
+
     def __repr__(self):
         return '{0}()'.format(self.__class__.__name__)
 

pywb/warcserver/test/test_handlers.py

@@ -1,4 +1,6 @@
+from gevent import monkey; monkey.patch_all()
 from .testutils import to_path, MementoOverrideTests, FakeRedisTests, BaseTestClass, TEST_CDX_PATH, TEST_WARC_PATH
+from .testutils import HttpBinLiveTests
 from collections import OrderedDict
 
 from io import BytesIO
@@ -42,7 +44,7 @@ ia_cdx = {
 
 
 
-class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass):
+class TestBaseWarcServer(HttpBinLiveTests, MementoOverrideTests, FakeRedisTests, BaseTestClass):
     @classmethod
     def setup_class(cls):
         super(TestBaseWarcServer, cls).setup_class()
@@ -142,6 +144,7 @@ class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass):
 
         cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')])
         cdxlist[0]['timestamp'] = '2016'
+        cdxlist[0]['load_url'] = 'http://httpbin.org/get'
         assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
                             'mime': '', 'load_url': 'http://httpbin.org/get',
                             'source': 'live', 'source-coll': 'live', 'timestamp': '2016'}])
@@ -286,7 +289,8 @@ Host: httpbin.org
         assert b'HTTP/1.1 200 OK' in resp.body
         assert b'"foo": "bar"' in resp.body
 
-        assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"}
+        #assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"}
+        assert "NotFoundException('http://webenact.rhizome.org/vvork/" in json.loads(resp.headers['ResErrors'])['rhiz']
 
     def test_agg_post_resolve_postreq(self):
         req_data = """\

pywb/warcserver/test/test_upstream.py

@@ -14,10 +14,10 @@ from pywb.warcserver.index.aggregator import SimpleAggregator
 
 from pywb.warcserver.upstreamindexsource import UpstreamMementoIndexSource, UpstreamAggIndexSource
 
-from .testutils import LiveServerTests, BaseTestClass
+from .testutils import LiveServerTests, HttpBinLiveTests, BaseTestClass
 
 
-class TestUpstream(LiveServerTests, BaseTestClass):
+class TestUpstream(LiveServerTests, HttpBinLiveTests, BaseTestClass):
     def setup(self):
         app = BaseWarcServer()
 

pywb/warcserver/test/testutils.py

@@ -142,7 +142,6 @@ class LiveServerTests(object):
     @classmethod
     def setup_class(cls):
         super(LiveServerTests, cls).setup_class()
-        #cls.server = ServerThreadRunner(cls.make_live_app())
         cls.server = GeventServer(cls.make_live_app())
 
     @staticmethod
@@ -157,5 +156,35 @@ class LiveServerTests(object):
 
     @classmethod
     def teardown_class(cls):
-        super(LiveServerTests, cls).teardown_class()
         cls.server.stop()
+        super(LiveServerTests, cls).teardown_class()
+
+
+# ============================================================================
+class HttpBinLiveTests(object):
+    @classmethod
+    def setup_class(cls, *args, **kwargs):
+        super(HttpBinLiveTests, cls).setup_class(*args, **kwargs)
+
+        from httpbin import app as httpbin_app
+        httpbin_app.config.update(JSONIFY_PRETTYPRINT_REGULAR=True)
+        cls.httpbin_server = GeventServer(httpbin_app)
+
+        httpbin_local = 'http://localhost:' + str(cls.httpbin_server.port) + '/'
+
+        def get_load_url(self, params):
+            params['url'] = params['url'].replace('http://httpbin.org/', httpbin_local)
+            params['url'] = params['url'].replace('https://httpbin.org/', httpbin_local)
+            return params['url']
+
+        cls.indexmock = patch('pywb.warcserver.index.indexsource.LiveIndexSource.get_load_url', get_load_url)
+        cls.indexmock.start()
+
+    @classmethod
+    def teardown_class(cls):
+        cls.indexmock.stop()
+        cls.httpbin_server.stop()
+        super(HttpBinLiveTests, cls).teardown_class()
+
+
+

requirements.txt

@@ -3,7 +3,7 @@ warcio>=1.5.0
 chardet
 requests
 redis
-jinja2<2.9
+jinja2
 surt>=0.3.0
 brotlipy
 pyyaml

setup.py

@@ -108,6 +108,7 @@ setup(
         'mock',
         'urllib3',
         'werkzeug',
+        'httpbin==0.5.0',
        ],
     cmdclass={'test': PyTest},
     test_suite='',

tests/test_live_rewriter.py

@@ -1,9 +1,10 @@
 from .base_config_test import BaseConfigTest, fmod_sl
+from pywb.warcserver.test.testutils import HttpBinLiveTests
 import pytest
 
 
 # ============================================================================
-class TestLiveRewriter(BaseConfigTest):
+class TestLiveRewriter(HttpBinLiveTests, BaseConfigTest):
     @classmethod
     def setup_class(cls):
         super(TestLiveRewriter, cls).setup_class('config_test.yaml')
@@ -26,13 +27,14 @@ class TestLiveRewriter(BaseConfigTest):
         assert resp.status_int == 200
 
     def test_live_anchor_encode(self, fmod_sl):
-        resp = self.get('/live/{0}httpbin.org/anything/abc%23%23xyz', fmod_sl)
+        resp = self.get('/live/{0}httpbin.org/get?val=abc%23%23xyz', fmod_sl)
-        assert '"http://httpbin.org/anything/abc##xyz"' in resp.text
+        assert 'get?val=abc%23%23xyz"' in resp.text
+        assert '"val": "abc##xyz"' in resp.text
+        #assert '"http://httpbin.org/anything/abc##xyz"' in resp.text
         assert resp.status_int == 200
 
     def test_live_head(self, fmod_sl):
-        resp = self.head('/live/{0}httpbin.org/anything/foo', fmod_sl)
+        resp = self.head('/live/{0}httpbin.org/get?foo=bar', fmod_sl)
-        #assert '"http://httpbin.org/anything/foo"' in resp.text
         assert resp.status_int == 200
 
     def test_live_live_frame(self):

tests/test_proxy.py

@@ -1,4 +1,4 @@
-from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests
+from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests, HttpBinLiveTests
 
 from .base_config_test import CollsDirMixin
 from pywb.utils.geventserver import GeventServer, RequestURIWSGIHandler
@@ -99,7 +99,7 @@ class TestProxy(BaseTestProxy):
 
 
 # ============================================================================
-class TestRecordingProxy(CollsDirMixin, BaseTestProxy):
+class TestRecordingProxy(HttpBinLiveTests, CollsDirMixin, BaseTestProxy):
     @classmethod
     def setup_class(cls, coll='pywb', config_file='config_test.yaml'):
         super(TestRecordingProxy, cls).setup_class('test', 'config_test_record.yaml', recording=True)

tests/test_record_replay.py

@@ -1,7 +1,7 @@
 from .base_config_test import BaseConfigTest, fmod, CollsDirMixin
 from pywb.manager.manager import main as manager
 from pywb.manager.autoindex import AutoIndexer
-from pywb.warcserver.test.testutils import to_path
+from pywb.warcserver.test.testutils import to_path, HttpBinLiveTests
 
 import os
 import time
@@ -9,7 +9,7 @@ import json
 
 
 # ============================================================================
-class TestRecordReplay(CollsDirMixin, BaseConfigTest):
+class TestRecordReplay(HttpBinLiveTests, CollsDirMixin, BaseConfigTest):
     @classmethod
     def setup_class(cls):
         super(TestRecordReplay, cls).setup_class('config_test_record.yaml')
@@ -133,7 +133,7 @@ class TestRecordReplay(CollsDirMixin, BaseConfigTest):
 
 
 # ============================================================================
-class TestRecordCustomConfig(CollsDirMixin, BaseConfigTest):
+class TestRecordCustomConfig(HttpBinLiveTests, CollsDirMixin, BaseConfigTest):
     @classmethod
     def setup_class(cls):
         rec_custom = {'recorder': {'source_coll': 'live',