From 82d3b61523e07eeb5de6c8e5019f265c41926be7 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 12 Aug 2016 01:19:30 -0400 Subject: [PATCH] recorder: catch exception in close_idle_files() if file no longer exists and ensure it's removed --- recorder/warcwriter.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/recorder/warcwriter.py b/recorder/warcwriter.py index e45cf408..b125da5e 100644 --- a/recorder/warcwriter.py +++ b/recorder/warcwriter.py @@ -376,8 +376,11 @@ class MultiFileWARCWriter(BaseWARCWriter): return fh def _close_file(self, fh): - fcntl.flock(fh, fcntl.LOCK_UN) - fh.close() + try: + fcntl.flock(fh, fcntl.LOCK_UN) + fh.close() + except Exception as e: + print(e) def get_dir_key(self, params): return res_template(self.key_template, params) @@ -506,8 +509,14 @@ class MultiFileWARCWriter(BaseWARCWriter): now = datetime.datetime.now() for dir_key, out, filename in self.iter_open_files(): - mtime = os.path.getmtime(filename) + try: + mtime = os.path.getmtime(filename) + except: + self.close_key(dir_key) + return + mtime = datetime.datetime.fromtimestamp(mtime) + if (now - mtime) > self.max_idle_time: print('Closing idle ' + filename) self.close_key(dir_key)