backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

recorder: close_file() by params rather than exact path, update tests

Browse Source
This commit is contained in:
Ilya Kreymer 2016-03-26 13:07:53 -04:00
parent 7deba42851
commit 7884d4394b
4 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
recorder/recorderapp.py
View File

@ -182,12 +182,6 @@ class Wrapper(object):
self.out.write(buff) self.out.write(buff)
return buff return buff
def close(self):
try:
self.stream.close()
except:
traceback.print_exc()
#============================================================================== #==============================================================================
class RespWrapper(Wrapper): class RespWrapper(Wrapper):
@ -248,4 +242,8 @@ class ReqWrapper(Wrapper):
if not n.upper().startswith('WARC-'): if not n.upper().startswith('WARC-'):
del self.headers[n] del self.headers[n]
def close(self):
# no need to close wsgi.input
pass

2
recorder/test/test_recorder.py
View File

@ -383,7 +383,7 @@ class TestRecorder(LiveServerTests, FakeRedisTests, TempDirTests, BaseTestClass)
assert len(writer.fh_cache) == 1 assert len(writer.fh_cache) == 1
writer.remove_file(self.root_dir + '/warcs/FOO/') writer.close_file({'param.recorder.coll': 'FOO'})
assert len(writer.fh_cache) == 0 assert len(writer.fh_cache) == 0

19
recorder/warcwriter.py
View File

@ -275,11 +275,15 @@ class MultiFileWARCWriter(BaseWARCWriter):
fcntl.flock(fh, fcntl.LOCK_UN) fcntl.flock(fh, fcntl.LOCK_UN)
fh.close() fh.close()
def remove_file(self, full_dir): def close_file(self, params):
full_dir = res_template(self.dir_template, params)
result = self.fh_cache.pop(full_dir, None) result = self.fh_cache.pop(full_dir, None)
if result: if not result:
out, filename = result return
self._close_file(out)
out, filename = result
self._close_file(out)
return filename
def _do_write_req_resp(self, req, resp, params): def _do_write_req_resp(self, req, resp, params):
full_dir = res_template(self.dir_template, params) full_dir = res_template(self.dir_template, params)
@ -323,10 +327,9 @@ class MultiFileWARCWriter(BaseWARCWriter):
close_file = True close_file = True
if close_file: if close_file:
if is_new: self._close_file(out)
self._close_file(out) if not is_new:
else: self.fh_cache.pop(full_dir, None)
self.remove_file(full_dir)
elif is_new: elif is_new:
fcntl.flock(out, fcntl.LOCK_EX | fcntl.LOCK_NB) fcntl.flock(out, fcntl.LOCK_EX | fcntl.LOCK_NB)

3
webagg/utils.py
View File

@ -140,8 +140,7 @@ def res_template(template, params, **extra_params):
formatter = params.get('_formatter') formatter = params.get('_formatter')
if not formatter: if not formatter:
formatter = ParamFormatter(params) formatter = ParamFormatter(params)
res = formatter.format(template, url=params.get('url', ''), **extra_params)
res = formatter.format(template, url=params['url'], **extra_params)
return res return res