backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

tests: fix all tests for updated to webenact, use https when possible for webenact and example page tests (#511)

Browse Source
This commit is contained in:
Ilya Kreymer 2019-10-26 09:03:25 +01:00 committed by GitHub
parent 9ce324212a
commit 59b735ee99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 44 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pywb/apps/test/test_rewriter.py
View File

@ -26,7 +26,7 @@ class TestRewriterApp(FakeRedisTests, BaseTestClass):
resp = self.testapp.get('/live/mp_/http://example.com/') resp = self.testapp.get('/live/mp_/http://example.com/')
resp.charset = 'utf-8' resp.charset = 'utf-8'
assert '"http://localhost:80/live/mp_/http://www.iana.org/domains/example"' in resp.text assert '"http://localhost:80/live/mp_/https://www.iana.org/domains/example"' in resp.text
assert '"http://example.com/"' assert '"http://example.com/"'

4
pywb/utils/test/test_loaders.py
View File

@ -23,11 +23,11 @@ IOError: [Errno 2] No such file or directory: '_x_no_such_file_'
'Example Domain' 'Example Domain'
# range request # range request
>>> print_str(BlockLoader().load('http://example.com', 1262).read()) >>> print_str(BlockLoader().load('http://example.com', 1248).read())
'</html>\n' '</html>\n'
# custom profile # custom profile
>>> print_str(BlockLoader().load('local+http://example.com', 1262).read()) >>> print_str(BlockLoader().load('local+http://example.com', 1248).read())
'</html>\n' '</html>\n'
# unknown loader error # unknown loader error

37
pywb/warcserver/index/test/test_indexsource.py
View File

@ -26,12 +26,12 @@ class TestIndexSources(FakeRedisTests, BaseTestClass):
cls.all_sources = { cls.all_sources = {
'file': FileIndexSource(TEST_CDX_PATH + 'iana.cdxj'), 'file': FileIndexSource(TEST_CDX_PATH + 'iana.cdxj'),
'redis': RedisIndexSource('redis://localhost:6379/2/test:rediscdx'), 'redis': RedisIndexSource('redis://localhost:6379/2/test:rediscdx'),
'remote_cdx': RemoteIndexSource('http://webenact.rhizome.org/all-cdx?url={url}', 'remote_cdx': RemoteIndexSource('https://webenact.rhizome.org/all/cdx?url={url}',
'http://webenact.rhizome.org/all/{timestamp}id_/{url}'), 'https://webenact.rhizome.org/all/{timestamp}id_/{url}'),
'memento': MementoIndexSource('http://webenact.rhizome.org/all/{url}', 'memento': MementoIndexSource('https://webenact.rhizome.org/all/{url}',
'http://webenact.rhizome.org/all/timemap/*/{url}', 'https://webenact.rhizome.org/all/timemap/link/{url}',
'http://webenact.rhizome.org/all/{timestamp}id_/{url}') 'https://webenact.rhizome.org/all/{timestamp}id_/{url}')
} }
@pytest.fixture(params=local_sources) @pytest.fixture(params=local_sources)
@ -99,11 +99,14 @@ org,iana)/domains/root/servers 20140126201227 iana.warc.gz"""
res, errs = self.query_single_source(remote_source, dict(url=url)) res, errs = self.query_single_source(remote_source, dict(url=url))
expected = """\ expected = """\
com,instagram)/amaliaulman 20141014150552 http://webenact.rhizome.org/all/20141014150552id_/http://instagram.com/amaliaulman com,instagram)/amaliaulman 20141014150552 https://webenact.rhizome.org/all/20141014150552id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014155217 http://webenact.rhizome.org/all/20141014155217id_/http://instagram.com/amaliaulman com,instagram)/amaliaulman 20141014152101 https://webenact.rhizome.org/all/20141014152101id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014162333 http://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman com,instagram)/amaliaulman 20141014155217 https://webenact.rhizome.org/all/20141014155217id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014171636 http://webenact.rhizome.org/all/20141014171636id_/http://instagram.com/amaliaulman""" com,instagram)/amaliaulman 20141014160238 https://webenact.rhizome.org/all/20141014160238id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014162333 https://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014163116 https://webenact.rhizome.org/all/20141014163116id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014171636 https://webenact.rhizome.org/all/20141014171636id_/http://instagram.com/amaliaulman
com,instagram)/amaliaulman 20141014171954 https://webenact.rhizome.org/all/20141014171954id_/http://instagram.com/amaliaulman"""
assert(key_ts_res(res, 'load_url') == expected) assert(key_ts_res(res, 'load_url') == expected)
assert(errs == {}) assert(errs == {})
@ -114,21 +117,21 @@ com,instagram)/amaliaulman 20141014171636 http://webenact.rhizome.org/all/201410
res, errs = self.query_single_source(remote_source, dict(url=url, closest='20141014162332', limit=1)) res, errs = self.query_single_source(remote_source, dict(url=url, closest='20141014162332', limit=1))
expected = """\ expected = """\
com,instagram)/amaliaulman 20141014162333 http://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman""" com,instagram)/amaliaulman 20141014162333 https://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman"""
assert(key_ts_res(res, 'load_url') == expected) assert(key_ts_res(res, 'load_url') == expected)
assert(errs == {}) assert(errs == {})
# Url Match -- Wb Memento # Url Match -- Wb Memento
def test_remote_closest_wb_memnto_loader(self): def test_remote_closest_wb_memnto_loader(self):
replay = 'http://webenact.rhizome.org/all/{timestamp}id_/{url}' replay = 'https://webenact.rhizome.org/all/{timestamp}id_/{url}'
source = WBMementoIndexSource(replay, '', replay) source = WBMementoIndexSource(replay, '', replay)
url = 'http://instagram.com/amaliaulman' url = 'http://instagram.com/amaliaulman'
res, errs = self.query_single_source(source, dict(url=url, closest='20141014162332', limit=1)) res, errs = self.query_single_source(source, dict(url=url, closest='20141014162332', limit=1))
expected = """\ expected = """\
com,instagram)/amaliaulman 20141014162333 http://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman""" com,instagram)/amaliaulman 20141014162333 https://webenact.rhizome.org/all/20141014162333id_/http://instagram.com/amaliaulman"""
assert(key_ts_res(res, 'load_url') == expected) assert(key_ts_res(res, 'load_url') == expected)
assert(errs == {}) assert(errs == {})
@ -151,20 +154,20 @@ com,instagram)/amaliaulman 20141014162333 http://webenact.rhizome.org/all/201410
expected = '' expected = ''
assert(key_ts_res(res) == expected) assert(key_ts_res(res) == expected)
if all_source == self.all_sources[remote_sources[0]]: if all_source == self.all_sources['memento']:
assert('http%3A//x-not-found-x.notfound/' in errs['source']) assert('x-not-found-x.notfound/' in errs['source'])
else: else:
assert(errs == {}) assert(errs == {})
def test_another_remote_not_found(self): def test_another_remote_not_found(self):
source = MementoIndexSource.from_timegate_url('http://webenact.rhizome.org/all/') source = MementoIndexSource.from_timegate_url('https://webenact.rhizome.org/all/')
url = 'http://x-not-found-x.notfound/' url = 'http://x-not-found-x.notfound/'
res, errs = self.query_single_source(source, dict(url=url, limit=3)) res, errs = self.query_single_source(source, dict(url=url, limit=3))
expected = '' expected = ''
assert(key_ts_res(res) == expected) assert(key_ts_res(res) == expected)
assert(errs['source'] == "NotFoundException('http://webenact.rhizome.org/all/timemap/link/http://x-not-found-x.notfound/',)") assert(errs['source'] == "NotFoundException('https://webenact.rhizome.org/all/timemap/link/http://x-not-found-x.notfound/',)")
def test_file_not_found(self): def test_file_not_found(self):
source = FileIndexSource('testdata/not-found-x') source = FileIndexSource('testdata/not-found-x')

22
pywb/warcserver/index/test/test_memento_agg.py
View File

@ -23,7 +23,7 @@ sources = {
'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'), 'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'),
'ait': MementoIndexSource.from_timegate_url('http://wayback.archive-it.org/all/'), 'ait': MementoIndexSource.from_timegate_url('http://wayback.archive-it.org/all/'),
'bl': MementoIndexSource.from_timegate_url('http://www.webarchive.org.uk/wayback/archive/'), 'bl': MementoIndexSource.from_timegate_url('http://www.webarchive.org.uk/wayback/archive/'),
'rhiz': MementoIndexSource.from_timegate_url('http://webenact.rhizome.org/vvork/', path='*') 'rhiz': MementoIndexSource.from_timegate_url('https://webenact.rhizome.org/vvork/', path='*')
} }
aggs = {'simple': SimpleAggregator(sources), aggs = {'simple': SimpleAggregator(sources),
@ -59,7 +59,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
assert(to_json_list(res) == exp) assert(to_json_list(res) == exp)
assert(errs == {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://iana.org/',)", assert(errs == {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://iana.org/',)",
'rhiz': "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"}) 'rhiz': "NotFoundException('https://webenact.rhizome.org/vvork/http://iana.org/',)"})
@pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys()))
@ -77,7 +77,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
] ]
assert(to_json_list(res) == exp) assert(to_json_list(res) == exp)
assert(errs == {'rhiz': "NotFoundException('http://webenact.rhizome.org/vvork/http://example.com/',)"}) assert(errs == {'rhiz': "NotFoundException('https://webenact.rhizome.org/vvork/http://example.com/',)"})
@pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys())) @pytest.mark.parametrize("agg", list(aggs.values()), ids=list(aggs.keys()))
@ -86,7 +86,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
url = 'http://vvork.com/' url = 'http://vvork.com/'
res, errs = agg(dict(url=url, closest='20141001', limit=5)) res, errs = agg(dict(url=url, closest='20141001', limit=5))
exp = [{"timestamp": "20141006184357", "load_url": "http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"}, exp = [{"timestamp": "20141006184357", "load_url": "https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"},
{"timestamp": "20141018133107", "load_url": "http://web.archive.org/web/20141018133107id_/http://vvork.com/", "source": "ia"}, {"timestamp": "20141018133107", "load_url": "http://web.archive.org/web/20141018133107id_/http://vvork.com/", "source": "ia"},
{"timestamp": "20141020161243", "load_url": "http://web.archive.org/web/20141020161243id_/http://vvork.com/", "source": "ia"}, {"timestamp": "20141020161243", "load_url": "http://web.archive.org/web/20141020161243id_/http://vvork.com/", "source": "ia"},
{"timestamp": "20140806161228", "load_url": "http://web.archive.org/web/20140806161228id_/http://vvork.com/", "source": "ia"}, {"timestamp": "20140806161228", "load_url": "http://web.archive.org/web/20140806161228id_/http://vvork.com/", "source": "ia"},
@ -102,7 +102,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
url = 'http://vvork.com/' url = 'http://vvork.com/'
res, errs = agg(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait')) res, errs = agg(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait'))
exp = [{"timestamp": "20141006184357", "load_url": "http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"}, exp = [{"timestamp": "20141006184357", "load_url": "https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"},
{"timestamp": "20131004231540", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"}] {"timestamp": "20131004231540", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"}]
assert(to_json_list(res) == exp) assert(to_json_list(res) == exp)
@ -167,7 +167,7 @@ class TestMemAgg(MementoOverrideTests, BaseTestClass):
headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait')) headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait'))
exp = b"""\ exp = b"""\
com,vvork)/ 20141006184357 {"url": "http://www.vvork.com/", "mem_rel": "memento", "memento_url": "http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/", "load_url": "http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"} com,vvork)/ 20141006184357 {"url": "http://www.vvork.com/", "mem_rel": "memento", "memento_url": "https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/", "load_url": "https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"}
com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento", "memento_url": "http://wayback.archive-it.org/all/20131004231540/http://vvork.com/", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"} com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento", "memento_url": "http://wayback.archive-it.org/all/20131004231540/http://vvork.com/", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"}
""" """
@ -183,7 +183,7 @@ com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento
headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='json')) headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='json'))
exp = b"""\ exp = b"""\
{"urlkey": "com,vvork)/", "timestamp": "20141006184357", "url": "http://www.vvork.com/", "mem_rel": "memento", "memento_url": "http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/", "load_url": "http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"} {"urlkey": "com,vvork)/", "timestamp": "20141006184357", "url": "http://www.vvork.com/", "mem_rel": "memento", "memento_url": "https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/", "load_url": "https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/", "source": "rhiz"}
{"urlkey": "com,vvork)/", "timestamp": "20131004231540", "url": "http://vvork.com/", "mem_rel": "last memento", "memento_url": "http://wayback.archive-it.org/all/20131004231540/http://vvork.com/", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"} {"urlkey": "com,vvork)/", "timestamp": "20131004231540", "url": "http://vvork.com/", "mem_rel": "last memento", "memento_url": "http://wayback.archive-it.org/all/20131004231540/http://vvork.com/", "load_url": "http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/", "source": "ait"}
""" """
@ -198,7 +198,7 @@ com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento
headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='link')) headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='link'))
exp = b"""\ exp = b"""\
<http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT"; src="rhiz", <https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT"; src="rhiz",
<http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/>; rel="memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT"; src="ait" <http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/>; rel="memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT"; src="ait"
""" """
assert(headers['Content-Type'] == 'application/link') assert(headers['Content-Type'] == 'application/link')
@ -223,7 +223,7 @@ com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento
assert(b''.join(res) == exp) assert(b''.join(res) == exp)
exp_errs = {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://iana.org/',)", exp_errs = {'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://iana.org/',)",
'rhiz': "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"} 'rhiz': "NotFoundException('https://webenact.rhizome.org/vvork/http://iana.org/',)"}
assert(errs == exp_errs) assert(errs == exp_errs)
@ -242,7 +242,7 @@ com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento
exp_errs = {'ait': "NotFoundException('http://wayback.archive-it.org/all/http://foo.bar.non-existent',)", exp_errs = {'ait': "NotFoundException('http://wayback.archive-it.org/all/http://foo.bar.non-existent',)",
'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://foo.bar.non-existent',)", 'bl': "NotFoundException('http://www.webarchive.org.uk/wayback/archive/http://foo.bar.non-existent',)",
'ia': "NotFoundException('http://web.archive.org/web/http://foo.bar.non-existent',)", 'ia': "NotFoundException('http://web.archive.org/web/http://foo.bar.non-existent',)",
'rhiz': "NotFoundException('http://webenact.rhizome.org/vvork/http://foo.bar.non-existent',)"} 'rhiz': "NotFoundException('https://webenact.rhizome.org/vvork/http://foo.bar.non-existent',)"}
assert(errs == exp_errs) assert(errs == exp_errs)
@ -253,7 +253,7 @@ com,vvork)/ 20131004231540 {"url": "http://vvork.com/", "mem_rel": "last memento
headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='text')) headers, res, errs = handler(dict(url=url, closest='20141001', limit=2, sources='rhiz,ait', output='text'))
exp = b"""\ exp = b"""\
com,vvork)/ 20141006184357 http://www.vvork.com/ memento http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/ http://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/ rhiz com,vvork)/ 20141006184357 http://www.vvork.com/ memento https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/ https://webenact.rhizome.org/vvork/20141006184357id_/http://www.vvork.com/ rhiz
com,vvork)/ 20131004231540 http://vvork.com/ last memento http://wayback.archive-it.org/all/20131004231540/http://vvork.com/ http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/ ait com,vvork)/ 20131004231540 http://vvork.com/ last memento http://wayback.archive-it.org/all/20131004231540/http://vvork.com/ http://wayback.archive-it.org/all/20131004231540id_/http://vvork.com/ ait
""" """
assert(headers['Content-Type'] == 'text/plain') assert(headers['Content-Type'] == 'text/plain')

10
pywb/warcserver/test/test_handlers.py
View File

@ -33,7 +33,7 @@ from pywb.utils.memento import MementoUtils
sources = { sources = {
'local': DirectoryIndexSource(TEST_CDX_PATH), 'local': DirectoryIndexSource(TEST_CDX_PATH),
'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'), 'ia': MementoIndexSource.from_timegate_url('http://web.archive.org/web/'),
'rhiz': MementoIndexSource.from_timegate_url('http://webenact.rhizome.org/vvork/', path='*'), 'rhiz': MementoIndexSource.from_timegate_url('https://webenact.rhizome.org/vvork/'),
'live': LiveIndexSource(), 'live': LiveIndexSource(),
} }
@ -247,7 +247,7 @@ class TestBaseWarcServer(HttpBinLiveTests, MementoOverrideTests, FakeRedisTests,
assert resp.headers['Link'] == MementoUtils.make_link('http://www.iana.org/', 'original') assert resp.headers['Link'] == MementoUtils.make_link('http://www.iana.org/', 'original')
assert resp.headers['Memento-Datetime'] == 'Sun, 26 Jan 2014 20:06:24 GMT' assert resp.headers['Memento-Datetime'] == 'Sun, 26 Jan 2014 20:06:24 GMT'
assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"} assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('https://webenact.rhizome.org/vvork/http://iana.org/',)"}
@patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_postreq')) @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_local_postreq'))
def test_agg_select_local_postreq(self): def test_agg_select_local_postreq(self):
@ -267,7 +267,7 @@ Host: iana.org
assert resp.headers['Link'] == MementoUtils.make_link('http://www.iana.org/', 'original') assert resp.headers['Link'] == MementoUtils.make_link('http://www.iana.org/', 'original')
assert resp.headers['Memento-Datetime'] == 'Sun, 26 Jan 2014 20:06:24 GMT' assert resp.headers['Memento-Datetime'] == 'Sun, 26 Jan 2014 20:06:24 GMT'
assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://iana.org/',)"} assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('https://webenact.rhizome.org/vvork/http://iana.org/',)"}
@patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live_postreq')) @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live_postreq'))
def test_agg_live_postreq(self): def test_agg_live_postreq(self):
@ -290,8 +290,8 @@ Host: httpbin.org
assert b'HTTP/1.1 200 OK' in resp.body assert b'HTTP/1.1 200 OK' in resp.body
assert b'"foo": "bar"' in resp.body assert b'"foo": "bar"' in resp.body
#assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('http://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"} #assert json.loads(resp.headers['ResErrors']) == {"rhiz": "NotFoundException('https://webenact.rhizome.org/vvork/http://httpbin.org/get?foo=bar',)"}
assert "NotFoundException('http://webenact.rhizome.org/vvork/" in json.loads(resp.headers['ResErrors'])['rhiz'] assert "NotFoundException('https://webenact.rhizome.org/vvork/" in json.loads(resp.headers['ResErrors'])['rhiz']
def test_agg_post_resolve_postreq(self): def test_agg_post_resolve_postreq(self):
req_data = """\ req_data = """\

10
sample_archive/text_content/link_headers.yaml
View File

@ -20,13 +20,13 @@ agg_test_3:
'http://wayback.archive-it.org/all/{url}': '<http://vvork.com/>; rel="original", <http://wayback.archive-it.org/all/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://wayback.archive-it.org/all/20131004231540/http://vvork.com/>; rel="last memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT", <http://wayback.archive-it.org/all/20090710005710/http://vvork.com/>; rel="first memento"; datetime="Fri, 10 Jul 2009 00:57:10 GMT", <http://wayback.archive-it.org/all/20131004175706/http://vvork.com/>; rel="prev memento"; datetime="Fri, 04 Oct 2013 17:57:06 GMT"' 'http://wayback.archive-it.org/all/{url}': '<http://vvork.com/>; rel="original", <http://wayback.archive-it.org/all/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://wayback.archive-it.org/all/20131004231540/http://vvork.com/>; rel="last memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT", <http://wayback.archive-it.org/all/20090710005710/http://vvork.com/>; rel="first memento"; datetime="Fri, 10 Jul 2009 00:57:10 GMT", <http://wayback.archive-it.org/all/20131004175706/http://vvork.com/>; rel="prev memento"; datetime="Fri, 04 Oct 2013 17:57:06 GMT"'
'http://webenact.rhizome.org/vvork/{url}': '<http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <http://webenact.rhizome.org/vvork/timemap/*/http://www.vvork.com/>; rel="timemap"; type="application/link-format"' 'https://webenact.rhizome.org/vvork/{url}': '<https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <https://webenact.rhizome.org/vvork/timemap/link/http://www.vvork.com/>; rel="timemap"; type="application/link-format"'
agg_test_4: agg_test_4:
'http://wayback.archive-it.org/all/{url}': '<http://vvork.com/>; rel="original", <http://wayback.archive-it.org/all/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://wayback.archive-it.org/all/20131004231540/http://vvork.com/>; rel="last memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT", <http://wayback.archive-it.org/all/20090710005710/http://vvork.com/>; rel="first memento"; datetime="Fri, 10 Jul 2009 00:57:10 GMT", <http://wayback.archive-it.org/all/20131004175706/http://vvork.com/>; rel="prev memento"; datetime="Fri, 04 Oct 2013 17:57:06 GMT"' 'http://wayback.archive-it.org/all/{url}': '<http://vvork.com/>; rel="original", <http://wayback.archive-it.org/all/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://wayback.archive-it.org/all/20131004231540/http://vvork.com/>; rel="last memento"; datetime="Fri, 04 Oct 2013 23:15:40 GMT", <http://wayback.archive-it.org/all/20090710005710/http://vvork.com/>; rel="first memento"; datetime="Fri, 10 Jul 2009 00:57:10 GMT", <http://wayback.archive-it.org/all/20131004175706/http://vvork.com/>; rel="prev memento"; datetime="Fri, 04 Oct 2013 17:57:06 GMT"'
'http://webenact.rhizome.org/vvork/{url}': '<http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <http://webenact.rhizome.org/vvork/timemap/*/http://www.vvork.com/>; rel="timemap"; type="application/link-format"' 'https://webenact.rhizome.org/vvork/{url}': '<https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <https://webenact.rhizome.org/vvork/timemap/link/http://www.vvork.com/>; rel="timemap"; type="application/link-format"'
agg_test_5: agg_test_5:
@ -36,11 +36,11 @@ agg_test_5:
select_mem_1: select_mem_1:
'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20140806161228/http://vvork.com/>; rel="prev memento"; datetime="Wed, 06 Aug 2014 16:12:28 GMT", <http://web.archive.org/web/20141018133107/http://vvork.com/>; rel="memento"; datetime="Sat, 18 Oct 2014 13:31:07 GMT", <http://web.archive.org/web/20141020161243/http://vvork.com/>; rel="next memento"; datetime="Mon, 20 Oct 2014 16:12:43 GMT", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT"' 'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20140806161228/http://vvork.com/>; rel="prev memento"; datetime="Wed, 06 Aug 2014 16:12:28 GMT", <http://web.archive.org/web/20141018133107/http://vvork.com/>; rel="memento"; datetime="Sat, 18 Oct 2014 13:31:07 GMT", <http://web.archive.org/web/20141020161243/http://vvork.com/>; rel="next memento"; datetime="Mon, 20 Oct 2014 16:12:43 GMT", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT"'
'http://webenact.rhizome.org/vvork/{url}': '<http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <http://webenact.rhizome.org/vvork/timemap/*/http://www.vvork.com/>; rel="timemap"; type="application/link-format"' 'https://webenact.rhizome.org/vvork/{url}': '<https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <https://webenact.rhizome.org/vvork/timemap/link/http://www.vvork.com/>; rel="timemap"; type="application/link-format"'
select_mem_2: select_mem_2:
'http://webenact.rhizome.org/vvork/{url}': '<http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <http://webenact.rhizome.org/vvork/timemap/*/http://www.vvork.com/>; rel="timemap"; type="application/link-format"' 'https://webenact.rhizome.org/vvork/{url}': '<https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <https://webenact.rhizome.org/vvork/timemap/link/http://www.vvork.com/>; rel="timemap"; type="application/link-format"'
'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20151105012627/http://vvork.com/>; rel="prev memento"; datetime="Thu, 05 Nov 2015 01:26:27 GMT", <http://web.archive.org/web/20160110134855/http://vvork.com/>; rel="memento"; datetime="Sun, 10 Jan 2016 13:48:55 GMT", <http://web.archive.org/web/20160112032847/http://vvork.com/>; rel="next memento"; datetime="Tue, 12 Jan 2016 03:28:47 GMT", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT"' 'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20151105012627/http://vvork.com/>; rel="prev memento"; datetime="Thu, 05 Nov 2015 01:26:27 GMT", <http://web.archive.org/web/20160110134855/http://vvork.com/>; rel="memento"; datetime="Sun, 10 Jan 2016 13:48:55 GMT", <http://web.archive.org/web/20160112032847/http://vvork.com/>; rel="next memento"; datetime="Tue, 12 Jan 2016 03:28:47 GMT", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT"'
@ -48,7 +48,7 @@ select_mem_2:
select_live: select_live:
'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20161011164443/http://vvork.com/>; rel="prev memento"; datetime="Tue, 11 Oct 2016 16:44:43 GMT"' 'http://web.archive.org/web/{url}': '<http://vvork.com/>; rel="original", <http://web.archive.org/web/timemap/link/http://vvork.com/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/20161027001353/http://vvork.com/>; rel="last memento"; datetime="Thu, 27 Oct 2016 00:13:53 GMT", <http://web.archive.org/web/20020727091331/http://vvork.com/>; rel="first memento"; datetime="Sat, 27 Jul 2002 09:13:31 GMT", <http://web.archive.org/web/20161011164443/http://vvork.com/>; rel="prev memento"; datetime="Tue, 11 Oct 2016 16:44:43 GMT"'
'http://webenact.rhizome.org/vvork/{url}': '<http://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <http://webenact.rhizome.org/vvork/timemap/*/http://www.vvork.com/>; rel="timemap"; type="application/link-format"' 'https://webenact.rhizome.org/vvork/{url}': '<https://webenact.rhizome.org/vvork/20141006184357/http://www.vvork.com/>; rel="memento"; datetime="Mon, 06 Oct 2014 18:43:57 GMT", <http://www.vvork.com/>; rel="original", <https://webenact.rhizome.org/vvork/timemap/link/http://www.vvork.com/>; rel="timemap"; type="application/link-format"'
select_local: select_local:
'http://web.archive.org/web/{url}': '<http://iana.org/>; rel="original", <http://web.archive.org/web/timemap/link/http://iana.org/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/19971210061738/http://iana.org/>; rel="first memento"; datetime="Wed, 10 Dec 1997 06:17:38 GMT", <http://web.archive.org/web/20140123034755/http://iana.org/>; rel="prev memento"; datetime="Thu, 23 Jan 2014 03:47:55 GMT", <http://web.archive.org/web/20140126093743/http://iana.org/>; rel="memento"; datetime="Sun, 26 Jan 2014 09:37:43 GMT", <http://web.archive.org/web/20140129175203/http://iana.org/>; rel="next memento"; datetime="Wed, 29 Jan 2014 17:52:03 GMT", <http://web.archive.org/web/20161114190210/http://iana.org/>; rel="last memento"; datetime="Mon, 14 Nov 2016 19:02:10 GMT"' 'http://web.archive.org/web/{url}': '<http://iana.org/>; rel="original", <http://web.archive.org/web/timemap/link/http://iana.org/>; rel="timemap"; type="application/link-format", <http://web.archive.org/web/19971210061738/http://iana.org/>; rel="first memento"; datetime="Wed, 10 Dec 1997 06:17:38 GMT", <http://web.archive.org/web/20140123034755/http://iana.org/>; rel="prev memento"; datetime="Thu, 23 Jan 2014 03:47:55 GMT", <http://web.archive.org/web/20140126093743/http://iana.org/>; rel="memento"; datetime="Sun, 26 Jan 2014 09:37:43 GMT", <http://web.archive.org/web/20140129175203/http://iana.org/>; rel="next memento"; datetime="Wed, 29 Jan 2014 17:52:03 GMT", <http://web.archive.org/web/20161114190210/http://iana.org/>; rel="last memento"; datetime="Mon, 14 Nov 2016 19:02:10 GMT"'