backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

Local httpbin tests + LiveIndexSource improvement (#318)

Browse Source 
tests and LiveIndexSource improvements:
- run local instance of httpbin in separate gevent server for any httpbin.org requests
- LiveIndexSource: has overridable get_load_url(), also use 'load_url' for HEAD check, remove unused proxy_url
- test update: add HttpBinLiveTests which patches LiveIndexSource.get_load_url() to redirect httpbin requests to local instance
- test update: just use httpbin.org/get instead of httpbin.org/anything, unsupported in older version (0.5.0) require for windows support
- setup: add 'httpbin==0.5.0' to test requires, remove jinja2 pin to old version
This commit is contained in:
Ilya Kreymer 2018-04-28 18:20:37 -07:00 committed by GitHub
parent de3ec0e1bc
commit bef63b4c6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 61 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pywb/recorder/test/test_recorder.py
View File

@ -2,7 +2,7 @@ from gevent import monkey; monkey.patch_all()
import gevent
from pywb.warcserver.test.testutils import TempDirTests, LiveServerTests, BaseTestClass, to_path
from pywb.warcserver.test.testutils import FakeRedisTests
from pywb.warcserver.test.testutils import FakeRedisTests, HttpBinLiveTests
import os
import webtest
@ -45,7 +45,7 @@ Cookie: boo=far\r\n\
class TestRecorder(LiveServerTests, FakeRedisTests, TempDirTests, BaseTestClass):
class TestRecorder(LiveServerTests, HttpBinLiveTests, FakeRedisTests, TempDirTests, BaseTestClass):
@classmethod
def setup_class(cls):
super(TestRecorder, cls).setup_class()

10
pywb/warcserver/index/indexsource.py
View File

@ -196,8 +196,7 @@ class RemoteIndexSource(BaseIndexSource):
#=============================================================================
class LiveIndexSource(BaseIndexSource):
def __init__(self, proxy_url='{url}'):
self.proxy_url = proxy_url
def __init__(self):
self._init_sesh(DefaultAdapters.live_adapter)
def load_index(self, params):
@ -209,14 +208,14 @@ class LiveIndexSource(BaseIndexSource):
cdx['urlkey'] = params.get('key').decode('utf-8')
cdx['timestamp'] = timestamp_now()
cdx['url'] = params['url']
cdx['load_url'] = res_template(self.proxy_url, params)
cdx['load_url'] = self.get_load_url(params)
cdx['is_live'] = 'true'
mime = params.get('content_type', '')
if params.get('filter') and not mime:
try:
res = self.sesh.head(cdx['url'])
res = self.sesh.head(cdx['load_url'])
if res.status_code != 405:
cdx['status'] = str(res.status_code)
@ -231,6 +230,9 @@ class LiveIndexSource(BaseIndexSource):
return iter([cdx])
def get_load_url(self, params):
return params['url']
def __repr__(self):
return '{0}()'.format(self.__class__.__name__)

8
pywb/warcserver/test/test_handlers.py
View File

@ -1,4 +1,6 @@
from gevent import monkey; monkey.patch_all()
from .testutils import to_path, MementoOverrideTests, FakeRedisTests, BaseTestClass, TEST_CDX_PATH, TEST_WARC_PATH
from .testutils import HttpBinLiveTests
from collections import OrderedDict
from io import BytesIO
@ -42,7 +44,7 @@ ia_cdx = {
class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass):
class TestBaseWarcServer(HttpBinLiveTests, MementoOverrideTests, FakeRedisTests, BaseTestClass):
@classmethod
def setup_class(cls):
super(TestBaseWarcServer, cls).setup_class()
@ -142,6 +144,7 @@ class TestBaseWarcServer(MementoOverrideTests, FakeRedisTests, BaseTestClass):
cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')])
cdxlist[0]['timestamp'] = '2016'
cdxlist[0]['load_url'] = 'http://httpbin.org/get'
assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
'mime': '', 'load_url': 'http://httpbin.org/get',
'source': 'live', 'source-coll': 'live', 'timestamp': '2016'}])
@ -286,7 +289,8 @@ Host: httpbin.org
assert b'HTTP/1.1 200 OK' in resp.body
assert b'"foo": "bar"' in resp.body
assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"}
#assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"}
assert "NotFoundException('http://webenact.rhizome.org/vvork/" in json.loads(resp.headers['ResErrors'])['rhiz']
def test_agg_post_resolve_postreq(self):
req_data = """\

4
pywb/warcserver/test/test_upstream.py
View File

@ -14,10 +14,10 @@ from pywb.warcserver.index.aggregator import SimpleAggregator
from pywb.warcserver.upstreamindexsource import UpstreamMementoIndexSource, UpstreamAggIndexSource
from .testutils import LiveServerTests, BaseTestClass
from .testutils import LiveServerTests, HttpBinLiveTests, BaseTestClass
class TestUpstream(LiveServerTests, BaseTestClass):
class TestUpstream(LiveServerTests, HttpBinLiveTests, BaseTestClass):
def setup(self):
app = BaseWarcServer()

33
pywb/warcserver/test/testutils.py
View File

@ -142,7 +142,6 @@ class LiveServerTests(object):
@classmethod
def setup_class(cls):
super(LiveServerTests, cls).setup_class()
#cls.server = ServerThreadRunner(cls.make_live_app())
cls.server = GeventServer(cls.make_live_app())
@staticmethod
@ -157,5 +156,35 @@ class LiveServerTests(object):
@classmethod
def teardown_class(cls):
super(LiveServerTests, cls).teardown_class()
cls.server.stop()
super(LiveServerTests, cls).teardown_class()
# ============================================================================
class HttpBinLiveTests(object):
@classmethod
def setup_class(cls, *args, **kwargs):
super(HttpBinLiveTests, cls).setup_class(*args, **kwargs)
from httpbin import app as httpbin_app
httpbin_app.config.update(JSONIFY_PRETTYPRINT_REGULAR=True)
cls.httpbin_server = GeventServer(httpbin_app)
httpbin_local = 'http://localhost:' + str(cls.httpbin_server.port) + '/'
def get_load_url(self, params):
params['url'] = params['url'].replace('http://httpbin.org/', httpbin_local)
params['url'] = params['url'].replace('https://httpbin.org/', httpbin_local)
return params['url']
cls.indexmock = patch('pywb.warcserver.index.indexsource.LiveIndexSource.get_load_url', get_load_url)
cls.indexmock.start()
@classmethod
def teardown_class(cls):
cls.indexmock.stop()
cls.httpbin_server.stop()
super(HttpBinLiveTests, cls).teardown_class()

2
requirements.txt
View File

@ -3,7 +3,7 @@ warcio>=1.5.0
chardet
requests
redis
jinja2<2.9
jinja2
surt>=0.3.0
brotlipy
pyyaml

1
setup.py
View File

@ -108,6 +108,7 @@ setup(
'mock',
'urllib3',
'werkzeug',
'httpbin==0.5.0',
],
cmdclass={'test': PyTest},
test_suite='',

12
tests/test_live_rewriter.py
View File

@ -1,9 +1,10 @@
from .base_config_test import BaseConfigTest, fmod_sl
from pywb.warcserver.test.testutils import HttpBinLiveTests
import pytest
# ============================================================================
class TestLiveRewriter(BaseConfigTest):
class TestLiveRewriter(HttpBinLiveTests, BaseConfigTest):
@classmethod
def setup_class(cls):
super(TestLiveRewriter, cls).setup_class('config_test.yaml')
@ -26,13 +27,14 @@ class TestLiveRewriter(BaseConfigTest):
assert resp.status_int == 200
def test_live_anchor_encode(self, fmod_sl):
resp = self.get('/live/{0}httpbin.org/anything/abc%23%23xyz', fmod_sl)
assert '"http://httpbin.org/anything/abc##xyz"' in resp.text
resp = self.get('/live/{0}httpbin.org/get?val=abc%23%23xyz', fmod_sl)
assert 'get?val=abc%23%23xyz"' in resp.text
assert '"val": "abc##xyz"' in resp.text
#assert '"http://httpbin.org/anything/abc##xyz"' in resp.text
assert resp.status_int == 200
def test_live_head(self, fmod_sl):
resp = self.head('/live/{0}httpbin.org/anything/foo', fmod_sl)
#assert '"http://httpbin.org/anything/foo"' in resp.text
resp = self.head('/live/{0}httpbin.org/get?foo=bar', fmod_sl)
assert resp.status_int == 200
def test_live_live_frame(self):

4
tests/test_proxy.py
View File

@ -1,4 +1,4 @@
from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests
from pywb.warcserver.test.testutils import BaseTestClass, TempDirTests, HttpBinLiveTests
from .base_config_test import CollsDirMixin
from pywb.utils.geventserver import GeventServer, RequestURIWSGIHandler
@ -99,7 +99,7 @@ class TestProxy(BaseTestProxy):
# ============================================================================
class TestRecordingProxy(CollsDirMixin, BaseTestProxy):
class TestRecordingProxy(HttpBinLiveTests, CollsDirMixin, BaseTestProxy):
@classmethod
def setup_class(cls, coll='pywb', config_file='config_test.yaml'):
super(TestRecordingProxy, cls).setup_class('test', 'config_test_record.yaml', recording=True)

6
tests/test_record_replay.py
View File

@ -1,7 +1,7 @@
from .base_config_test import BaseConfigTest, fmod, CollsDirMixin
from pywb.manager.manager import main as manager
from pywb.manager.autoindex import AutoIndexer
from pywb.warcserver.test.testutils import to_path
from pywb.warcserver.test.testutils import to_path, HttpBinLiveTests
import os
import time
@ -9,7 +9,7 @@ import json
# ============================================================================
class TestRecordReplay(CollsDirMixin, BaseConfigTest):
class TestRecordReplay(HttpBinLiveTests, CollsDirMixin, BaseConfigTest):
@classmethod
def setup_class(cls):
super(TestRecordReplay, cls).setup_class('config_test_record.yaml')
@ -133,7 +133,7 @@ class TestRecordReplay(CollsDirMixin, BaseConfigTest):
# ============================================================================
class TestRecordCustomConfig(CollsDirMixin, BaseConfigTest):
class TestRecordCustomConfig(HttpBinLiveTests, CollsDirMixin, BaseConfigTest):
@classmethod
def setup_class(cls):
rec_custom = {'recorder': {'source_coll': 'live',