backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-28 00:25:21 +01:00
Code Issues Projects Releases Activity

redis: fix redis key lookup, add tests for zrangebylex with new fakeredis

Browse Source
This commit is contained in:
Ilya Kreymer 2016-03-09 18:33:04 -08:00
parent 0198bf1213
commit c1bdeac92b
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pywb/cdx/cdxsource.py
View File

@ -131,8 +131,8 @@ class RedisCDXSource(CDXSource):
def load_sorted_range(self, query, cdx_key): def load_sorted_range(self, query, cdx_key):
cdx_list = self.redis.zrangebylex(cdx_key, cdx_list = self.redis.zrangebylex(cdx_key,
'[' + query.key, b'[' + query.key,
'(' + query.end_key) b'(' + query.end_key)
return iter(cdx_list) return iter(cdx_list)
@ -143,7 +143,7 @@ class RedisCDXSource(CDXSource):
# key is not part of list, so prepend to each line # key is not part of list, so prepend to each line
key += b' ' key += b' '
cdx_list = map(lambda x: key + x, cdx_list) cdx_list = list(map(lambda x: key + x, cdx_list))
return cdx_list return cdx_list
def __str__(self): def __str__(self):

9
pywb/cdx/test/test_redis_source.py
View File

@ -4,8 +4,10 @@ com,example)/ 20130729195151 http://test@example.com/ warc/revisit - B2LTWWPUOYA
com,example)/ 20140127171200 http://example.com text/html 200 B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 1046 334 dupes.warc.gz com,example)/ 20140127171200 http://example.com text/html 200 B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 1046 334 dupes.warc.gz
com,example)/ 20140127171251 http://example.com warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 553 11875 dupes.warc.gz com,example)/ 20140127171251 http://example.com warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 553 11875 dupes.warc.gz
# TODO: enable when FakeRedis supports zrangebylex! >>> redis_cdx(redis_cdx_server_key, 'http://example.com')
#>>> redis_cdx(redis_cdx_server_key, 'http://example.com') com,example)/ 20130729195151 http://test@example.com/ warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 591 355 example-url-agnostic-revisit.warc.gz
com,example)/ 20140127171200 http://example.com text/html 200 B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 1046 334 dupes.warc.gz
com,example)/ 20140127171251 http://example.com warc/revisit - B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A - - 553 11875 dupes.warc.gz
""" """
@ -21,8 +23,7 @@ from pywb import get_test_dir
import sys import sys
import os import os
test_cdx_dir = get_test_dir() + 'cdx/' test_cdx_dir = os.path.join(get_test_dir(), 'cdx/')
def load_cdx_into_redis(source, filename, key=None): def load_cdx_into_redis(source, filename, key=None):
# load a cdx into mock redis # load a cdx into mock redis

3
setup.py
View File

@ -90,6 +90,9 @@ setup(
'fakeredis', 'fakeredis',
'mock', 'mock',
], ],
dependency_links=[
'git+https://github.com/ikreymer/fakeredis.git@zset-lex-ops#egg=fakeredis-0.6.2-zset-lex-ops',
],
cmdclass={'test': PyTest}, cmdclass={'test': PyTest},
test_suite='', test_suite='',
entry_points=""" entry_points="""