backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

zipnum: backport fix for #173, paths specified in a zipnum .loc file are relative to the .loc file, not to

Browse Source 
the working dir of the application
warnings: don't warn on .gz cdx files
This commit is contained in:
Ilya Kreymer 2016-03-31 13:09:57 -07:00
parent ba7ac56230
commit e5ef51363c
5 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pywb/cdx/cdxserver.py
View File

@ -176,8 +176,8 @@ class CDXServer(BaseCDXServer):
if filename.endswith(('.summary', '.idx')): if filename.endswith(('.summary', '.idx')):
return ZipNumCluster(filename, config) return ZipNumCluster(filename, config)
# no warning for .loc # no warning for .loc or .gz (zipnum)
if not filename.endswith('.loc'): if not filename.endswith(('.loc', '.gz')):
logging.warn('skipping unrecognized URI: %s', filename) logging.warn('skipping unrecognized URI: %s', filename)
return None return None

2
pywb/cdx/test/test_zipnum.py
View File

@ -149,7 +149,7 @@ Traceback (most recent call last):
Exception: No Locations Found for: foo Exception: No Locations Found for: foo
>>> zip_test_err(url='http://iana.org/x', matchType='exact') >>> zip_test_err(url='http://iana.org/x', matchType='exact') # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last): Traceback (most recent call last):
IOError: [Errno 2] No such file or directory: './sample_archive/invalid' IOError: [Errno 2] No such file or directory: './sample_archive/invalid'

11
pywb/cdx/zipnum.py
View File

@ -51,11 +51,20 @@ class LocMapResolver(object):
# update loc file mtime # update loc file mtime
self.loc_mtime = new_mtime self.loc_mtime = new_mtime
local_dir = os.path.dirname(self.loc_filename)
def res_path(pathname):
if '://' not in pathname:
pathname = os.path.join(local_dir, pathname)
return pathname
logging.debug('Loading loc from: ' + self.loc_filename) logging.debug('Loading loc from: ' + self.loc_filename)
with open(self.loc_filename, 'rb') as fh: with open(self.loc_filename, 'rb') as fh:
for line in fh: for line in fh:
parts = line.rstrip().split('\t') parts = line.rstrip().split('\t')
self.loc_map[parts[0]] = parts[1:]
paths = [res_path(pathname) for pathname in parts[1:]]
self.loc_map[parts[0]] = paths
def __call__(self, part, query): def __call__(self, part, query):
return self.loc_map[part] return self.loc_map[part]

4
sample_archive/zipcdx/zipnum-bad.loc
View File

@ -1,3 +1,3 @@
bar ./sample_archive/invalid bar invalid
foo2 foo2
zipnum ./sample_archive/x-bad-path-to-ignore-x ./sample_archive/zipcdx/zipnum-sample.cdx.gz zipnum x-bad-path-to-ignore-x zipnum-sample.cdx.gz

2
sample_archive/zipcdx/zipnum-sample.loc
View File

@ -1 +1 @@
zipnum ./sample_archive/x-bad-path-to-ignore-x ./sample_archive/zipcdx/zipnum-sample.cdx.gz zipnum x-bad-path-to-ignore-x zipnum-sample.cdx.gz