From 8a7ed0cf574eedc442c47a2f77d52de26287acf3 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 28 Feb 2018 11:45:10 -0800 Subject: [PATCH 1/2] bump dev version number after merge --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4b80e1..266f2df 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev152', + version='2.4b2.dev153', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', From c2172c6b5bf7a9c8f7241ef59f3fdd31a92313b3 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 28 Feb 2018 13:02:03 -0800 Subject: [PATCH 2/2] make sure to roll over idle warcs even when warcprox is idle itself --- setup.py | 2 +- warcprox/writerthread.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 266f2df..5169090 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev153', + version='2.4b2.dev154', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index 1010161..b3dceab 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -45,9 +45,12 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.batch = set() def _get_process_put(self): - recorded_url = self.inq.get(block=True, timeout=0.5) - self.batch.add(recorded_url) - self.pool.submit(self._process_url, recorded_url) + try: + recorded_url = self.inq.get(block=True, timeout=0.5) + self.batch.add(recorded_url) + self.pool.submit(self._process_url, recorded_url) + finally: + self.writer_pool.maybe_idle_rollover() def _process_url(self, recorded_url): try: @@ -67,7 +70,6 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.batch.remove(recorded_url) if self.outq: self.outq.put(recorded_url) - self.writer_pool.maybe_idle_rollover() def _filter_accepts(self, recorded_url): if not self.method_filter: