recorder: close_file() by params rather than exact path, update tests

recorder/recorderapp.py

@@ -182,12 +182,6 @@ class Wrapper(object):
         self.out.write(buff)
         return buff
 
-    def close(self):
-        try:
-            self.stream.close()
-        except:
-            traceback.print_exc()
-
 
 #==============================================================================
 class RespWrapper(Wrapper):
@@ -248,4 +242,8 @@ class ReqWrapper(Wrapper):
             if not n.upper().startswith('WARC-'):
                 del self.headers[n]
 
+    def close(self):
+        # no need to close wsgi.input
+        pass
+
 

recorder/test/test_recorder.py

@@ -383,7 +383,7 @@ class TestRecorder(LiveServerTests, FakeRedisTests, TempDirTests, BaseTestClass)
 
         assert len(writer.fh_cache) == 1
 
-        writer.remove_file(self.root_dir + '/warcs/FOO/')
+        writer.close_file({'param.recorder.coll': 'FOO'})
 
         assert len(writer.fh_cache) == 0
 

recorder/warcwriter.py

@@ -275,11 +275,15 @@ class MultiFileWARCWriter(BaseWARCWriter):
         fcntl.flock(fh, fcntl.LOCK_UN)
         fh.close()
 
-    def remove_file(self, full_dir):
+    def close_file(self, params):
+        full_dir = res_template(self.dir_template, params)
         result = self.fh_cache.pop(full_dir, None)
-        if result:
-            out, filename = result
-            self._close_file(out)
+        if not result:
+            return
+
+        out, filename = result
+        self._close_file(out)
+        return filename
 
     def _do_write_req_resp(self, req, resp, params):
         full_dir = res_template(self.dir_template, params)
@@ -323,10 +327,9 @@ class MultiFileWARCWriter(BaseWARCWriter):
                 close_file = True
 
             if close_file:
-                if is_new:
-                    self._close_file(out)
-                else:
-                    self.remove_file(full_dir)
+                self._close_file(out)
+                if not is_new:
+                    self.fh_cache.pop(full_dir, None)
 
             elif is_new:
                 fcntl.flock(out, fcntl.LOCK_EX | fcntl.LOCK_NB)

webagg/utils.py

@@ -140,8 +140,7 @@ def res_template(template, params, **extra_params):
     formatter = params.get('_formatter')
     if not formatter:
         formatter = ParamFormatter(params)
-
-    res = formatter.format(template, url=params['url'], **extra_params)
+    res = formatter.format(template, url=params.get('url', ''), **extra_params)
 
     return res