backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

make warcprox finish writing all urls in the queue before shutting down

Browse Source
This commit is contained in:
Noah Levitt 2015-10-28 21:39:17 +00:00
parent 1b8d83203c
commit e0fe06c891
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
warcprox/writerthread.py
View File

@ -1,5 +1,3 @@
# vim:set sw=4 et:
from __future__ import absolute_import from __future__ import absolute_import
try: try:
@ -38,9 +36,8 @@ class WarcWriterThread(threading.Thread):
def run(self): def run(self):
try: try:
# XXX warcprox can shut down with urls to archive left in the queue
self.setName('WarcWriterThread(tid={})'.format(warcprox.gettid())) self.setName('WarcWriterThread(tid={})'.format(warcprox.gettid()))
while not self.stop.is_set(): while True:
try: try:
recorded_url = self.recorded_url_q.get(block=True, timeout=0.5) recorded_url = self.recorded_url_q.get(block=True, timeout=0.5)
self.idle = None self.idle = None
@ -54,6 +51,8 @@ class WarcWriterThread(threading.Thread):
if recorded_url.response_recorder and recorded_url.response_recorder.tempfile: if recorded_url.response_recorder and recorded_url.response_recorder.tempfile:
recorded_url.response_recorder.tempfile.close() recorded_url.response_recorder.tempfile.close()
except queue.Empty: except queue.Empty:
if self.stop.is_set():
break
self.idle = time.time() self.idle = time.time()
self.writer_pool.maybe_idle_rollover() self.writer_pool.maybe_idle_rollover()