backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

recorder: use res_template() to resolve params, rename indexing method to add_urls_to_index

Browse Source
This commit is contained in:
Ilya Kreymer 2016-03-23 23:55:21 -04:00
parent aa80cd6881
commit ba66d0bb5e
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
recorder/redisindexer.py
View File

@ -32,7 +32,7 @@ class WritableRedisIndexer(RedisIndexSource):
self.redis.hset(file_key, filename, full_filename) self.redis.hset(file_key, filename, full_filename)
def index_records(self, stream, params, filename=None): def add_urls_to_index(self, stream, params, filename=None):
rel_path = res_template(self.rel_path_template, params) rel_path = res_template(self.rel_path_template, params)
filename = os.path.relpath(filename, rel_path) filename = os.path.relpath(filename, rel_path)

8
recorder/warcwriter.py
View File

@ -233,7 +233,6 @@ class Digester(object):
# ============================================================================ # ============================================================================
class MultiFileWARCWriter(BaseWARCWriter): class MultiFileWARCWriter(BaseWARCWriter):
def __init__(self, dir_template, filename_template=None, max_size=0, def __init__(self, dir_template, filename_template=None, max_size=0,
*args, **kwargs): *args, **kwargs):
super(MultiFileWARCWriter, self).__init__(*args, **kwargs) super(MultiFileWARCWriter, self).__init__(*args, **kwargs)
@ -254,8 +253,9 @@ class MultiFileWARCWriter(BaseWARCWriter):
def _open_file(self, dir_, params): def _open_file(self, dir_, params):
timestamp = timestamp20_now() timestamp = timestamp20_now()
filename = dir_ + self.filename_template.format(hostname=self.hostname, filename = dir_ + res_template(self.filename_template, params,
timestamp=timestamp) hostname=self.hostname,
timestamp=timestamp)
path, name = os.path.split(filename) path, name = os.path.split(filename)
@ -311,7 +311,7 @@ class MultiFileWARCWriter(BaseWARCWriter):
out.seek(start) out.seek(start)
if self.dedup_index: if self.dedup_index:
self.dedup_index.index_records(out, params, filename=filename) self.dedup_index.add_urls_to_index(out, params, filename=filename)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()