From 6536516375e65841531ad3fc2e687828d887804a Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Fri, 20 Sep 2019 12:49:09 +0000 Subject: [PATCH 1/2] Handle ValueError when trying to close WARC file We get a lot of the following error in production and warcprox becomes totally unresponsive when this happens. ``` CRITICAL:warcprox.writerthread.WarcWriterProcessor:WarcWriterProcessor(tid=16646) will try to continue after unexpected error Traceback (most recent call last): File "/opt/spn2/lib/python3.5/site-packages/warcprox/__init__.py", line 140, in _run self._get_process_put() File "/opt/spn2/lib/python3.5/site-packages/warcprox/writerthread.py", line 60, in _get_process_put self.writer_pool.maybe_idle_rollover() File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 233, in maybe_idle_rollover w.maybe_idle_rollover() File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 188, in maybe_idle_rollover self.close() File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 169, in close fcntl.lockf(self.f, fcntl.LOCK_UN) ValueError: I/O operation on closed file ``` Current code handles `IOError`. We also need to handle `ValueError` to address this. --- warcprox/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warcprox/writer.py b/warcprox/writer.py index 730d606..639e96d 100644 --- a/warcprox/writer.py +++ b/warcprox/writer.py @@ -167,7 +167,7 @@ class WarcWriter: if self.open_suffix == '': try: fcntl.lockf(self.f, fcntl.LOCK_UN) - except IOError as exc: + except (IOError, ValueError) as exc: self.logger.error( 'could not unlock file %s (%s)', self.path, exc) self.f.close() From a09901dcefe8060bb2802d7937aec8570048d4c5 Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Sat, 21 Sep 2019 09:43:27 +0000 Subject: [PATCH 2/2] Use "except Exception" to catch all exception types --- warcprox/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warcprox/writer.py b/warcprox/writer.py index 639e96d..5187d08 100644 --- a/warcprox/writer.py +++ b/warcprox/writer.py @@ -167,7 +167,7 @@ class WarcWriter: if self.open_suffix == '': try: fcntl.lockf(self.f, fcntl.LOCK_UN) - except (IOError, ValueError) as exc: + except Exception as exc: self.logger.error( 'could not unlock file %s (%s)', self.path, exc) self.f.close()