diff --git a/setup.py b/setup.py index c4b80e1..5169090 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev152', + version='2.4b2.dev154', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/writerthread.py b/warcprox/writerthread.py index 4492965..a3eeedb 100644 --- a/warcprox/writerthread.py +++ b/warcprox/writerthread.py @@ -45,9 +45,12 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.batch = set() def _get_process_put(self): - recorded_url = self.inq.get(block=True, timeout=0.5) - self.batch.add(recorded_url) - self.pool.submit(self._process_url, recorded_url) + try: + recorded_url = self.inq.get(block=True, timeout=0.5) + self.batch.add(recorded_url) + self.pool.submit(self._process_url, recorded_url) + finally: + self.writer_pool.maybe_idle_rollover() def _process_url(self, recorded_url): try: @@ -67,7 +70,6 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor): self.batch.remove(recorded_url) if self.outq: self.outq.put(recorded_url) - self.writer_pool.maybe_idle_rollover() def _filter_accepts(self, recorded_url): if not self.method_filter: