From 8a7ed0cf574eedc442c47a2f77d52de26287acf3 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Wed, 28 Feb 2018 11:45:10 -0800
Subject: [PATCH 1/2] bump dev version number after merge
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index c4b80e1..266f2df 100755
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,7 @@ except:
setuptools.setup(
name='warcprox',
- version='2.4b2.dev152',
+ version='2.4b2.dev153',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',
From c2172c6b5bf7a9c8f7241ef59f3fdd31a92313b3 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Wed, 28 Feb 2018 13:02:03 -0800
Subject: [PATCH 2/2] make sure to roll over idle warcs
even when warcprox is idle itself
---
setup.py | 2 +-
warcprox/writerthread.py | 10 ++++++----
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/setup.py b/setup.py
index 266f2df..5169090 100755
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,7 @@ except:
setuptools.setup(
name='warcprox',
- version='2.4b2.dev153',
+ version='2.4b2.dev154',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',
diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py
index 1010161..b3dceab 100644
--- a/warcprox/writerthread.py
+++ b/warcprox/writerthread.py
@@ -45,9 +45,12 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.batch = set()
def _get_process_put(self):
- recorded_url = self.inq.get(block=True, timeout=0.5)
- self.batch.add(recorded_url)
- self.pool.submit(self._process_url, recorded_url)
+ try:
+ recorded_url = self.inq.get(block=True, timeout=0.5)
+ self.batch.add(recorded_url)
+ self.pool.submit(self._process_url, recorded_url)
+ finally:
+ self.writer_pool.maybe_idle_rollover()
def _process_url(self, recorded_url):
try:
@@ -67,7 +70,6 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.batch.remove(recorded_url)
if self.outq:
self.outq.put(recorded_url)
- self.writer_pool.maybe_idle_rollover()
def _filter_accepts(self, recorded_url):
if not self.method_filter: