diff --git a/pywb/recorder/test/test_recorder.py b/pywb/recorder/test/test_recorder.py index 3780a69b..d206b3b2 100644 --- a/pywb/recorder/test/test_recorder.py +++ b/pywb/recorder/test/test_recorder.py @@ -2,7 +2,7 @@ from gevent import monkey; monkey.patch_all() import gevent from pywb.warcserver.test.testutils import TempDirTests, LiveServerTests, BaseTestClass, to_path -from pywb.warcserver.test.testutils import FakeRedisTests +from pywb.warcserver.test.testutils import FakeRedisTests, HttpBinLiveTests import os import webtest @@ -45,7 +45,7 @@ Cookie: boo=far\r\n\ -class TestRecorder(LiveServerTests, FakeRedisTests, TempDirTests, BaseTestClass): +class TestRecorder(LiveServerTests, HttpBinLiveTests, FakeRedisTests, TempDirTests, BaseTestClass): @classmethod def setup_class(cls): super(TestRecorder, cls).setup_class() diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 1250f42c..80baa77f 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -196,8 +196,7 @@ class RemoteIndexSource(BaseIndexSource): #============================================================================= class LiveIndexSource(BaseIndexSource): - def __init__(self, proxy_url='{url}'): - self.proxy_url = proxy_url + def __init__(self): self._init_sesh(DefaultAdapters.live_adapter) def load_index(self, params): @@ -209,14 +208,14 @@ class LiveIndexSource(BaseIndexSource): cdx['urlkey'] = params.get('key').decode('utf-8') cdx['timestamp'] = timestamp_now() cdx['url'] = params['url'] - cdx['load_url'] = res_template(self.proxy_url, params) + cdx['load_url'] = self.get_load_url(params) cdx['is_live'] = 'true' mime = params.get('content_type', '') if params.get('filter') and not mime: try: - res = self.sesh.head(cdx['url']) + res = self.sesh.head(cdx['load_url']) if res.status_code != 405: cdx['status'] = str(res.status_code) @@ -231,6 +230,9 @@ class LiveIndexSource(BaseIndexSource): return iter([cdx]) + def get_load_url(self, params): + return params['url'] + def __repr__(self): return '{0}()'.format(self.__class__.__name__) diff --git a/pywb/warcserver/test/test_handlers.py b/pywb/warcserver/test/test_handlers.py index c4b509e9..374b79b7 100644 --- a/pywb/warcserver/test/test_handlers.py +++ b/pywb/warcserver/test/test_handlers.py @@ -1,4 +1,6 @@ +from gevent import monkey; monkey.patch_all() from .testutils import to_path, MementoOverrideTests, FakeRedisTests, BaseTestClass, TEST_CDX_PATH, TEST_WARC_PATH +from .testutils import HttpBinLiveTests from collections import OrderedDict from io import BytesIO @@ -42,7 +44,7 @@ ia_cdx = { -class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass): +class TestBaseWarcServer(HttpBinLiveTests, MementoOverrideTests, FakeRedisTests, BaseTestClass): @classmethod def setup_class(cls): super(TestBaseWarcServer, cls).setup_class() @@ -142,6 +144,7 @@ class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass): cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')]) cdxlist[0]['timestamp'] = '2016' + cdxlist[0]['load_url'] = 'http://httpbin.org/get' assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true', 'mime': '', 'load_url': 'http://httpbin.org/get', 'source': 'live', 'source-coll': 'live', 'timestamp': '2016'}]) @@ -286,7 +289,8 @@ Host: httpbin.org assert b'HTTP/1.1 200 OK' in resp.body assert b'"foo": "bar"' in resp.body - assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"} + #assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"} + assert "NotFoundException('http://webenact.rhizome.org/vvork/" in json.loads(resp.headers['ResErrors'])['rhiz'] def test_agg_post_resolve_postreq(self): req_data = """\ diff --git a/pywb/warcserver/test/test_upstream.py b/pywb/warcserver/test/test_upstream.py index b66d68a6..1e8e93ed 100644 --- a/pywb/warcserver/test/test_upstream.py +++ b/pywb/warcserver/test/test_upstream.py @@ -14,10 +14,10 @@ from pywb.warcserver.index.aggregator import SimpleAggregator from pywb.warcserver.upstreamindexsource import UpstreamMementoIndexSource, UpstreamAggIndexSource -from .testutils import LiveServerTests, BaseTestClass +from .testutils import LiveServerTests, HttpBinLiveTests, BaseTestClass -class TestUpstream(LiveServerTests, BaseTestClass): +class TestUpstream(LiveServerTests, HttpBinLiveTests, BaseTestClass): def setup(self): app = BaseWarcServer() diff --git a/pywb/warcserver/test/testutils.py b/pywb/warcserver/test/testutils.py index a4082e22..1da837d5 100644 --- a/pywb/warcserver/test/testutils.py +++ b/pywb/warcserver/test/testutils.py @@ -142,7 +142,6 @@ class LiveServerTests(object): @classmethod def setup_class(cls): super(LiveServerTests, cls).setup_class() - #cls.server = ServerThreadRunner(cls.make_live_app()) cls.server = GeventServer(cls.make_live_app()) @staticmethod @@ -157,5 +156,35 @@ class LiveServerTests(object): @classmethod def teardown_class(cls): - super(LiveServerTests, cls).teardown_class() cls.server.stop() + super(LiveServerTests, cls).teardown_class() + + +# ============================================================================ +class HttpBinLiveTests(object): + @classmethod + def setup_class(cls, *args, **kwargs): + super(HttpBinLiveTests, cls).setup_class(*args, **kwargs) + + from httpbin import app as httpbin_app + httpbin_app.config.update(JSONIFY_PRETTYPRINT_REGULAR=True) + cls.httpbin_server = GeventServer(httpbin_app) + + httpbin_local = 'http://localhost:' + str(cls.httpbin_server.port) + '/' + + def get_load_url(self, params): + params['url'] = params['url'].replace('http://httpbin.org/', httpbin_local) + params['url'] = params['url'].replace('https://httpbin.org/', httpbin_local) + return params['url'] + + cls.indexmock = patch('pywb.warcserver.index.indexsource.LiveIndexSource.get_load_url', get_load_url) + cls.indexmock.start() + + @classmethod + def teardown_class(cls): + cls.indexmock.stop() + cls.httpbin_server.stop() + super(HttpBinLiveTests, cls).teardown_class() + + + diff --git a/requirements.txt b/requirements.txt index 1c5ee468..4702d5df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ warcio>=1.5.0 chardet requests redis -jinja2<2.9 +jinja2 surt>=0.3.0 brotlipy pyyaml diff --git a/setup.py b/setup.py index ed2de024..a2312fff 100755 --- a/setup.py +++ b/setup.py @@ -108,6 +108,7 @@ setup( 'mock', 'urllib3', 'werkzeug', + 'httpbin==0.5.0', ], cmdclass={'test': PyTest}, test_suite='', diff --git a/tests/test_live_rewriter.py b/tests/test_live_rewriter.py index 64140fb4..cd6afb71 100644 --- a/tests/test_live_rewriter.py +++ b/tests/test_live_rewriter.py @@ -1,9 +1,10 @@ from .base_config_test import BaseConfigTest, fmod_sl +from pywb.warcserver.test.testutils import HttpBinLiveTests import pytest # ============================================================================ -class TestLiveRewriter(BaseConfigTest): +class TestLiveRewriter(HttpBinLiveTests, BaseConfigTest): @classmethod def setup_class(cls): super(TestLiveRewriter, cls).setup_class('config_test.yaml') @@ -26,13 +27,14 @@ class TestLiveRewriter(BaseConfigTest): assert resp.status_int == 200 def test_live_anchor_encode(self, fmod_sl): - resp = self.get('/live/{0}httpbin.org/anything/abc%23%23xyz', fmod_sl) - assert '"http://httpbin.org/anything/abc##xyz"' in resp.text + resp = self.get('/live/{0}httpbin.org/get?val=abc%23%23xyz', fmod_sl) + assert 'get?val=abc%23%23xyz"' in resp.text + assert '"val": "abc##xyz"' in resp.text + #assert '"http://httpbin.org/anything/abc##xyz"' in resp.text assert resp.status_int == 200 def test_live_head(self, fmod_sl): - resp = self.head('/live/{0}httpbin.org/anything/foo', fmod_sl) - #assert '"http://httpbin.org/anything/foo"' in resp.text + resp = self.head('/live/{0}httpbin.org/get?foo=bar', fmod_sl) assert resp.status_int == 200 def test_live_live_frame(self): diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 0fee6b1a..517db992 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -1,4 +1,4 @@ -from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests +from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests, HttpBinLiveTests from .base_config_test import CollsDirMixin from pywb.utils.geventserver import GeventServer, RequestURIWSGIHandler @@ -99,7 +99,7 @@ class TestProxy(BaseTestProxy): # ============================================================================ -class TestRecordingProxy(CollsDirMixin, BaseTestProxy): +class TestRecordingProxy(HttpBinLiveTests, CollsDirMixin, BaseTestProxy): @classmethod def setup_class(cls, coll='pywb', config_file='config_test.yaml'): super(TestRecordingProxy, cls).setup_class('test', 'config_test_record.yaml', recording=True) diff --git a/tests/test_record_replay.py b/tests/test_record_replay.py index e5558fbb..9caa2e80 100644 --- a/tests/test_record_replay.py +++ b/tests/test_record_replay.py @@ -1,7 +1,7 @@ from .base_config_test import BaseConfigTest, fmod, CollsDirMixin from pywb.manager.manager import main as manager from pywb.manager.autoindex import AutoIndexer -from pywb.warcserver.test.testutils import to_path +from pywb.warcserver.test.testutils import to_path, HttpBinLiveTests import os import time @@ -9,7 +9,7 @@ import json # ============================================================================ -class TestRecordReplay(CollsDirMixin, BaseConfigTest): +class TestRecordReplay(HttpBinLiveTests, CollsDirMixin, BaseConfigTest): @classmethod def setup_class(cls): super(TestRecordReplay, cls).setup_class('config_test_record.yaml') @@ -133,7 +133,7 @@ class TestRecordReplay(CollsDirMixin, BaseConfigTest): # ============================================================================ -class TestRecordCustomConfig(CollsDirMixin, BaseConfigTest): +class TestRecordCustomConfig(HttpBinLiveTests, CollsDirMixin, BaseConfigTest): @classmethod def setup_class(cls): rec_custom = {'recorder': {'source_coll': 'live',