backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

make sure to roll over idle warcs

Browse Source 
even when warcprox is idle itself
This commit is contained in:
Noah Levitt 2018-02-28 13:02:03 -08:00
parent 8a7ed0cf57
commit c2172c6b5b
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev153', version='2.4b2.dev154',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

10
warcprox/writerthread.py
View File

@ -45,9 +45,12 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.batch = set() self.batch = set()
def _get_process_put(self): def _get_process_put(self):
recorded_url = self.inq.get(block=True, timeout=0.5) try:
self.batch.add(recorded_url) recorded_url = self.inq.get(block=True, timeout=0.5)
self.pool.submit(self._process_url, recorded_url) self.batch.add(recorded_url)
self.pool.submit(self._process_url, recorded_url)
finally:
self.writer_pool.maybe_idle_rollover()
def _process_url(self, recorded_url): def _process_url(self, recorded_url):
try: try:
@ -67,7 +70,6 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
self.batch.remove(recorded_url) self.batch.remove(recorded_url)
if self.outq: if self.outq:
self.outq.put(recorded_url) self.outq.put(recorded_url)
self.writer_pool.maybe_idle_rollover()
def _filter_accepts(self, recorded_url): def _filter_accepts(self, recorded_url):
if not self.method_filter: if not self.method_filter: