backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

redis: fix redis key lookup, add tests for zrangebylex with new fakeredis

Browse Source
This commit is contained in:
Ilya Kreymer 2016-03-09 18:33:04 -08:00
parent 0198bf1213
commit c1bdeac92b
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pywb/cdx/cdxsource.py
View File

@ -131,8 +131,8 @@ class RedisCDXSource(CDXSource):
def load_sorted_range(self, query, cdx_key):
cdx_list = self.redis.zrangebylex(cdx_key,
'[' + query.key,
'(' + query.end_key)
b'[' + query.key,
b'(' + query.end_key)
return iter(cdx_list)
@ -143,7 +143,7 @@ class RedisCDXSource(CDXSource):
# key is not part of list, so prepend to each line
key += b' '
cdx_list = map(lambda x: key + x, cdx_list)
cdx_list = list(map(lambda x: key + x, cdx_list))
return cdx_list
def __str__(self):

9
pywb/cdx/test/test_redis_source.py
View File

@ -4,8 +4,10 @@ com,example)/ 20130729195151 http://test@example.com/ warc/revisit - B2LTWWPUOYA
com,example)/ 20140127171200 http://example.com text/html 200 B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 1046 334 dupes.warc.gz
com,example)/ 20140127171251 http://example.com warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 553 11875 dupes.warc.gz
# TODO: enable when FakeRedis supports zrangebylex!
#>>> redis_cdx(redis_cdx_server_key, 'http://example.com')
>>> redis_cdx(redis_cdx_server_key, 'http://example.com')
com,example)/ 20130729195151 http://test@example.com/ warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 591 355 example-url-agnostic-revisit.warc.gz
com,example)/ 20140127171200 http://example.com text/html 200 B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 1046 334 dupes.warc.gz
com,example)/ 20140127171251 http://example.com warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 553 11875 dupes.warc.gz
"""
@ -21,8 +23,7 @@ from pywb import get_test_dir
import sys
import os
test_cdx_dir = get_test_dir() + 'cdx/'
test_cdx_dir = os.path.join(get_test_dir(), 'cdx/')
def load_cdx_into_redis(source, filename, key=None):
# load a cdx into mock redis

3
setup.py
View File

@ -90,6 +90,9 @@ setup(
'fakeredis',
'mock',
],
dependency_links=[
'git+https://github.com/ikreymer/fakeredis.git@zset-lex-ops#egg=fakeredis-0.6.2-zset-lex-ops',
],
cmdclass={'test': PyTest},
test_suite='',
entry_points="""