backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

reorganize WarcproxController.run_until_shutdown, moving parts of it into new start() and shutdown() methods, for easier integration into a separate python program

Browse Source
This commit is contained in:
Noah Levitt 2016-06-27 14:18:21 -05:00
parent fabd732b7f
commit 6410e4c8c7
3 changed files with 74 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
setup.py
View File

@ -1,37 +1,36 @@
#!/usr/bin/env python #!/usr/bin/env python
# '''
# setup.py - setuptools installation config for warcprox setup.py - setuptools installation configuration for warcprox
#
# Copyright (C) 2013-2016 Internet Archive Copyright (C) 2013-2016 Internet Archive
#
# This program is free software; you can redistribute it and/or This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2 as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version. of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA. USA.
# '''
from setuptools.command.test import test as TestCommand
import sys import sys
import setuptools import setuptools
# special class needs to be added to support the pytest written dump-anydbm tests # special class needs to be added to support the pytest written dump-anydbm tests
class PyTest(TestCommand): class PyTest(setuptools.command.test.TestCommand):
def finalize_options(self): def finalize_options(self):
TestCommand.finalize_options(self) setuptools.command.test.TestCommand.finalize_options(self)
self.test_args = [] self.test_args = []
self.test_suite = True self.test_suite = True
def run_tests(self): def run_tests(self):
#import here, cause outside the eggs aren't loaded # import here, because outside the eggs aren't loaded
import pytest import pytest
errno = pytest.main(self.test_args) errno = pytest.main(self.test_args)
sys.exit(errno) sys.exit(errno)
@ -50,7 +49,7 @@ except:
deps.append('futures') deps.append('futures')
setuptools.setup(name='warcprox', setuptools.setup(name='warcprox',
version='2.0.dev10', version='2.0.dev11',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

78
warcprox/controller.py
View File

@ -146,27 +146,60 @@ class WarcproxController(object):
warcprox.TRACE, "status in service registry: %s", warcprox.TRACE, "status in service registry: %s",
self.status_info) self.status_info)
def run_until_shutdown(self): def start(self):
""" # XXX check if already started
Start warcprox and run until shut down. Call
warcprox_controller.stop.set() to initiate graceful shutdown.
"""
if self.proxy.stats_db: if self.proxy.stats_db:
self.proxy.stats_db.start() self.proxy.stats_db.start()
proxy_thread = threading.Thread( self.proxy_thread = threading.Thread(
target=self.proxy.serve_forever, name='ProxyThread') target=self.proxy.serve_forever, name='ProxyThread')
proxy_thread.start() self.proxy_thread.start()
if self.warc_writer_thread.dedup_db: if self.warc_writer_thread.dedup_db:
self.warc_writer_thread.dedup_db.start() self.warc_writer_thread.dedup_db.start()
self.warc_writer_thread.start() self.warc_writer_thread.start()
if self.playback_proxy is not None: if self.playback_proxy is not None:
playback_proxy_thread = threading.Thread(target=self.playback_proxy.serve_forever, name='PlaybackProxyThread') self.playback_proxy_thread = threading.Thread(
target=self.playback_proxy.serve_forever,
name='PlaybackProxyThread')
playback_proxy_thread.start() playback_proxy_thread.start()
self.stop = threading.Event() self.stop = threading.Event()
def shutdown(self):
# XXX check if already shut down
self.warc_writer_thread.stop.set()
self.proxy.shutdown()
self.proxy.server_close()
if self.playback_proxy is not None:
self.playback_proxy.shutdown()
self.playback_proxy.server_close()
if self.playback_proxy.playback_index_db is not None:
self.playback_proxy.playback_index_db.close()
# wait for threads to finish
self.warc_writer_thread.join()
if self.proxy.stats_db:
self.proxy.stats_db.stop()
if self.warc_writer_thread.dedup_db:
self.warc_writer_thread.dedup_db.close()
self.proxy_thread.join()
if self.playback_proxy is not None:
self.playback_proxy_thread.join()
if self.service_registry and hasattr(self, "status_info"):
self.service_registry.unregister(self.status_info["id"])
def run_until_shutdown(self):
"""
Start warcprox and run until shut down. Call
warcprox_controller.stop.set() to initiate graceful shutdown.
"""
self.start()
last_mem_dbg = datetime.datetime.utcfromtimestamp(0) last_mem_dbg = datetime.datetime.utcfromtimestamp(0)
try: try:
@ -190,31 +223,10 @@ class WarcproxController(object):
time.sleep(0.5) time.sleep(0.5)
except: except:
self.logger.critical("fatal exception, shutting down", exc_info=True) self.logger.critical(
"shutting down in response to fatal exception",
exc_info=True)
pass pass
finally: finally:
self.warc_writer_thread.stop.set() self.shutdown()
self.proxy.shutdown()
self.proxy.server_close()
if self.playback_proxy is not None:
self.playback_proxy.shutdown()
self.playback_proxy.server_close()
if self.playback_proxy.playback_index_db is not None:
self.playback_proxy.playback_index_db.close()
# wait for threads to finish
self.warc_writer_thread.join()
if self.proxy.stats_db:
self.proxy.stats_db.stop()
if self.warc_writer_thread.dedup_db:
self.warc_writer_thread.dedup_db.close()
proxy_thread.join()
if self.playback_proxy is not None:
playback_proxy_thread.join()
if self.service_registry and hasattr(self, "status_info"):
self.service_registry.unregister(self.status_info["id"])

9
warcprox/main.py
View File

@ -209,14 +209,15 @@ def init_controller(args):
warc_writer_thread, playback_proxy, service_registry=svcreg, warc_writer_thread, playback_proxy, service_registry=svcreg,
options=options) options=options)
signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set())
signal.signal(signal.SIGINT, lambda a,b: controller.stop.set())
signal.signal(signal.SIGQUIT, dump_state)
return controller return controller
def real_main(args): def real_main(args):
controller = init_controller(args) controller = init_controller(args)
signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set())
signal.signal(signal.SIGINT, lambda a,b: controller.stop.set())
signal.signal(signal.SIGQUIT, dump_state)
controller.run_until_shutdown() controller.run_until_shutdown()
def parse_args(argv=sys.argv): def parse_args(argv=sys.argv):