backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

zipnum: add file mtime check to location loading #17

Browse Source
This commit is contained in:
Ilya Kreymer 2014-02-22 18:28:09 -08:00
parent d8d7435d77
commit a4f1224d16
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pywb/cdx/zipnum.py
View File

@ -53,8 +53,7 @@ class ZipNumCluster(CDXSource):
loc = config.get('zipnum_loc') loc = config.get('zipnum_loc')
cookie_maker = config.get('cookie_maker') cookie_maker = config.get('cookie_maker')
self.max_blocks = config.get('max_blocks', self.max_blocks = config.get('max_blocks', self.max_blocks)
self.max_blocks)
reload_ival = config.get('reload_interval', reload_ival) reload_ival = config.get('reload_interval', reload_ival)
@ -67,6 +66,7 @@ class ZipNumCluster(CDXSource):
# initial loc map # initial loc map
self.loc_map = {} self.loc_map = {}
self.loc_mtime = 0
self.load_loc() self.load_loc()
# reload interval # reload interval
@ -76,6 +76,14 @@ class ZipNumCluster(CDXSource):
self.blk_loader = BlockLoader(cookie_maker=cookie_maker) self.blk_loader = BlockLoader(cookie_maker=cookie_maker)
def load_loc(self): def load_loc(self):
# check modified time of current file before loading
new_mtime = os.path.getmtime(self.loc_filename)
if (new_mtime == self.loc_mtime):
return
# update loc file mtime
self.loc_mtime = new_mtime
logging.debug('Loading loc from: ' + self.loc_filename) logging.debug('Loading loc from: ' + self.loc_filename)
with open(self.loc_filename) as fh: with open(self.loc_filename) as fh:
for line in fh: for line in fh:
@ -91,13 +99,13 @@ class ZipNumCluster(CDXSource):
return None return None
def reload_loc(self): def reload_loc(self):
newtime = self.reload_timed(self.loc_update_time, reload_time = self.reload_timed(self.loc_update_time,
self.loc_map, self.loc_map,
self.reload_interval, self.reload_interval,
self.load_loc) self.load_loc)
if newtime: if reload_time:
self.loc_update_time = newtime self.loc_update_time = reload_time
def lookup_loc(self, part): def lookup_loc(self, part):
return self.loc_map[part] return self.loc_map[part]