From 6536516375e65841531ad3fc2e687828d887804a Mon Sep 17 00:00:00 2001
From: Vangelis Banos <vangelis@archive.org>
Date: Fri, 20 Sep 2019 12:49:09 +0000
Subject: [PATCH 1/2] Handle ValueError when trying to close WARC file
We get a lot of the following error in production and warcprox becomes
totally unresponsive when this happens.
```
CRITICAL:warcprox.writerthread.WarcWriterProcessor:WarcWriterProcessor(tid=16646) will try to continue after unexpected error
Traceback (most recent call last):
File "/opt/spn2/lib/python3.5/site-packages/warcprox/__init__.py", line 140, in _run
self._get_process_put()
File "/opt/spn2/lib/python3.5/site-packages/warcprox/writerthread.py", line 60, in _get_process_put
self.writer_pool.maybe_idle_rollover()
File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 233, in maybe_idle_rollover
w.maybe_idle_rollover()
File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 188, in maybe_idle_rollover
self.close()
File "/opt/spn2/lib/python3.5/site-packages/warcprox/writer.py", line 169, in close
fcntl.lockf(self.f, fcntl.LOCK_UN)
ValueError: I/O operation on closed file
```
Current code handles `IOError`. We also need to handle `ValueError` to address this.
---
warcprox/writer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/warcprox/writer.py b/warcprox/writer.py
index 730d606..639e96d 100644
--- a/warcprox/writer.py
+++ b/warcprox/writer.py
@@ -167,7 +167,7 @@ class WarcWriter:
if self.open_suffix == '':
try:
fcntl.lockf(self.f, fcntl.LOCK_UN)
- except IOError as exc:
+ except (IOError, ValueError) as exc:
self.logger.error(
'could not unlock file %s (%s)', self.path, exc)
self.f.close()
From a09901dcefe8060bb2802d7937aec8570048d4c5 Mon Sep 17 00:00:00 2001
From: Vangelis Banos <vangelis@archive.org>
Date: Sat, 21 Sep 2019 09:43:27 +0000
Subject: [PATCH 2/2] Use "except Exception" to catch all exception types
---
warcprox/writer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/warcprox/writer.py b/warcprox/writer.py
index 639e96d..5187d08 100644
--- a/warcprox/writer.py
+++ b/warcprox/writer.py
@@ -167,7 +167,7 @@ class WarcWriter:
if self.open_suffix == '':
try:
fcntl.lockf(self.f, fcntl.LOCK_UN)
- except (IOError, ValueError) as exc:
+ except Exception as exc:
self.logger.error(
'could not unlock file %s (%s)', self.path, exc)
self.f.close()