backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

fixes for windows:

Browse Source 
indexing: ensure '/' always written to cdx
autoindex: improved test case, ensure threads exit with join
style: fix long lines
This commit is contained in:
Ilya Kreymer 2015-03-25 10:56:53 -07:00
parent a7307a6d98
commit 90eee03cdb
4 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pywb/manager/autoindex.py
View File

@ -33,8 +33,10 @@ class CDXAutoIndexer(RegexMatchingEventHandler):
while keep_running: while keep_running:
time.sleep(sleep_time) time.sleep(sleep_time)
except KeyboardInterrupt: # pragma: no cover except KeyboardInterrupt: # pragma: no cover
observer.stop() pass
observer.join() finally:
self.observer.stop()
self.observer.join()
#============================================================================= #=============================================================================

10
pywb/manager/manager.py
View File

@ -249,7 +249,8 @@ directory structure expected by pywb
'a collection name: template <coll> --{1} {0}') 'a collection name: template <coll> --{1} {0}')
raise IOError(msg.format(template_name, verb)) raise IOError(msg.format(template_name, verb))
full_path = os.path.join(self.templates_dir, os.path.basename(filename)) full_path = os.path.join(self.templates_dir,
os.path.basename(filename))
except KeyError: except KeyError:
try: try:
@ -278,7 +279,8 @@ directory structure expected by pywb
fh.write(data) fh.write(data)
full_path = os.path.abspath(full_path) full_path = os.path.abspath(full_path)
print('Copied default template "{0}" to "{1}"'.format(filename, full_path)) msg = 'Copied default template "{0}" to "{1}"'
print(msg.format(filename, full_path))
def remove_template(self, template_name, force=False): def remove_template(self, template_name, force=False):
full_path, filename = self._get_template_path(template_name, 'remove') full_path, filename = self._get_template_path(template_name, 'remove')
@ -330,7 +332,9 @@ directory structure expected by pywb
def do_index(warc): def do_index(warc):
if any_coll: if any_coll:
coll_name = warc.split(self.colls_dir + os.path.sep)[-1].split('/')[0] coll_name = warc.split(self.colls_dir + os.path.sep)
coll_name = coll_name[-1].split(os.path.sep)[0]
if coll_name != self.coll_name: if coll_name != self.coll_name:
self._set_coll_dirs(coll_name) self._set_coll_dirs(coll_name)

15
pywb/warc/cdxindexer.py
View File

@ -143,6 +143,14 @@ class SortedCDXWriter(BaseCDXWriter):
ALLOWED_EXT = ('.arc', '.arc.gz', '.warc', '.warc.gz') ALLOWED_EXT = ('.arc', '.arc.gz', '.warc', '.warc.gz')
#=================================================================
def _resolve_rel_path(path, rel_root):
path = os.path.relpath(path, rel_root)
if os.path.sep != '/': #pragma: no cover
path = path.replace(os.path.sep, '/')
return path
#================================================================= #=================================================================
def iter_file_or_dir(inputs, recursive=True, rel_root=None): def iter_file_or_dir(inputs, recursive=True, rel_root=None):
for input_ in inputs: for input_ in inputs:
@ -150,7 +158,7 @@ def iter_file_or_dir(inputs, recursive=True, rel_root=None):
if not rel_root: if not rel_root:
filename = os.path.basename(input_) filename = os.path.basename(input_)
else: else:
filename = os.path.relpath(input_, rel_root) filename = _resolve_rel_path(input_, rel_root)
yield input_, filename yield input_, filename
@ -159,7 +167,7 @@ def iter_file_or_dir(inputs, recursive=True, rel_root=None):
if filename.endswith(ALLOWED_EXT): if filename.endswith(ALLOWED_EXT):
full_path = os.path.join(input_, filename) full_path = os.path.join(input_, filename)
if rel_root: if rel_root:
filename = os.path.relpath(full_path, rel_root) filename = _resolve_rel_path(full_path, rel_root)
yield full_path, filename yield full_path, filename
else: else:
@ -169,8 +177,7 @@ def iter_file_or_dir(inputs, recursive=True, rel_root=None):
full_path = os.path.join(root, filename) full_path = os.path.join(root, filename)
if not rel_root: if not rel_root:
rel_root = input_ rel_root = input_
rel_path = os.path.relpath(full_path, rel_root) rel_path = _resolve_rel_path(full_path, rel_root)
rel_path = rel_path.replace(os.path.sep, '/')
yield full_path, rel_path yield full_path, rel_path

6
tests/test_auto_colls.py
View File

@ -482,6 +482,8 @@ class TestManagedColls(object):
main(['autoindex']) main(['autoindex'])
thread.join()
index_file = os.path.join(auto_dir, INDEX_DIR, AUTOINDEX_FILE) index_file = os.path.join(auto_dir, INDEX_DIR, AUTOINDEX_FILE)
assert os.path.isfile(index_file) assert os.path.isfile(index_file)
@ -504,7 +506,9 @@ class TestManagedColls(object):
main(['autoindex', 'auto']) main(['autoindex', 'auto'])
# assert file was update thread.join()
# assert file was update
assert os.path.getmtime(index_file) > mtime assert os.path.getmtime(index_file) > mtime
def test_err_template_remove(self): def test_err_template_remove(self):