backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

webagg: redis lookup: if url contains wildcard, scan redis keys to check multiple keys until one is found

Browse Source 
webagg tests: fix test to include mime in live cdx
This commit is contained in:
Ilya Kreymer 2016-06-07 12:54:28 -04:00
parent d7c74b68de
commit 3fec766e39
2 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
webagg/responseloader.py
View File

@ -133,7 +133,17 @@ class RedisResolver(RedisIndexSource):
if hasattr(cdx, '_formatter') and cdx._formatter: if hasattr(cdx, '_formatter') and cdx._formatter:
redis_key = cdx._formatter.format(redis_key) redis_key = cdx._formatter.format(redis_key)
res = self.redis.hget(redis_key, filename) res = None
if '*' in redis_key:
for key in self.redis.scan_iter(redis_key):
#key = key.decode('utf-8')
res = self.redis.hget(key, filename)
if res:
break
else:
res = self.redis.hget(redis_key, filename)
if res and six.PY3: if res and six.PY3:
res = res.decode('utf-8') res = res.decode('utf-8')

2
webagg/test/test_handlers.py
View File

@ -124,7 +124,7 @@ class TestResAgg(FakeRedisTests, BaseTestClass):
cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')]) cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')])
cdxlist[0]['timestamp'] = '2016' cdxlist[0]['timestamp'] = '2016'
assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true', assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}]) 'mime': '', 'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}])
def test_live_resource(self): def test_live_resource(self):
headers = {'foo': 'bar'} headers = {'foo': 'bar'}