diff --git a/setup.py b/setup.py index 5dcfc75..e8e43e8 100755 --- a/setup.py +++ b/setup.py @@ -1,37 +1,36 @@ #!/usr/bin/env python -# -# setup.py - setuptools installation config for warcprox -# -# Copyright (C) 2013-2016 Internet Archive -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. -# +''' +setup.py - setuptools installation configuration for warcprox + +Copyright (C) 2013-2016 Internet Archive + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +USA. +''' -from setuptools.command.test import test as TestCommand import sys import setuptools # special class needs to be added to support the pytest written dump-anydbm tests -class PyTest(TestCommand): +class PyTest(setuptools.command.test.TestCommand): def finalize_options(self): - TestCommand.finalize_options(self) + setuptools.command.test.TestCommand.finalize_options(self) self.test_args = [] self.test_suite = True def run_tests(self): - #import here, cause outside the eggs aren't loaded + # import here, because outside the eggs aren't loaded import pytest errno = pytest.main(self.test_args) sys.exit(errno) @@ -50,7 +49,7 @@ except: deps.append('futures') setuptools.setup(name='warcprox', - version='2.0.dev10', + version='2.0.dev11', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/controller.py b/warcprox/controller.py index 540371a..b56ad50 100644 --- a/warcprox/controller.py +++ b/warcprox/controller.py @@ -146,27 +146,60 @@ class WarcproxController(object): warcprox.TRACE, "status in service registry: %s", self.status_info) - def run_until_shutdown(self): - """ - Start warcprox and run until shut down. Call - warcprox_controller.stop.set() to initiate graceful shutdown. - """ + def start(self): + # XXX check if already started if self.proxy.stats_db: self.proxy.stats_db.start() - proxy_thread = threading.Thread( + self.proxy_thread = threading.Thread( target=self.proxy.serve_forever, name='ProxyThread') - proxy_thread.start() + self.proxy_thread.start() if self.warc_writer_thread.dedup_db: self.warc_writer_thread.dedup_db.start() self.warc_writer_thread.start() if self.playback_proxy is not None: - playback_proxy_thread = threading.Thread(target=self.playback_proxy.serve_forever, name='PlaybackProxyThread') + self.playback_proxy_thread = threading.Thread( + target=self.playback_proxy.serve_forever, + name='PlaybackProxyThread') playback_proxy_thread.start() self.stop = threading.Event() + def shutdown(self): + # XXX check if already shut down + self.warc_writer_thread.stop.set() + self.proxy.shutdown() + self.proxy.server_close() + + if self.playback_proxy is not None: + self.playback_proxy.shutdown() + self.playback_proxy.server_close() + if self.playback_proxy.playback_index_db is not None: + self.playback_proxy.playback_index_db.close() + + # wait for threads to finish + self.warc_writer_thread.join() + + if self.proxy.stats_db: + self.proxy.stats_db.stop() + if self.warc_writer_thread.dedup_db: + self.warc_writer_thread.dedup_db.close() + + self.proxy_thread.join() + if self.playback_proxy is not None: + self.playback_proxy_thread.join() + + if self.service_registry and hasattr(self, "status_info"): + self.service_registry.unregister(self.status_info["id"]) + + def run_until_shutdown(self): + """ + Start warcprox and run until shut down. Call + warcprox_controller.stop.set() to initiate graceful shutdown. + """ + self.start() + last_mem_dbg = datetime.datetime.utcfromtimestamp(0) try: @@ -190,31 +223,10 @@ class WarcproxController(object): time.sleep(0.5) except: - self.logger.critical("fatal exception, shutting down", exc_info=True) + self.logger.critical( + "shutting down in response to fatal exception", + exc_info=True) pass finally: - self.warc_writer_thread.stop.set() - self.proxy.shutdown() - self.proxy.server_close() - - if self.playback_proxy is not None: - self.playback_proxy.shutdown() - self.playback_proxy.server_close() - if self.playback_proxy.playback_index_db is not None: - self.playback_proxy.playback_index_db.close() - - # wait for threads to finish - self.warc_writer_thread.join() - - if self.proxy.stats_db: - self.proxy.stats_db.stop() - if self.warc_writer_thread.dedup_db: - self.warc_writer_thread.dedup_db.close() - - proxy_thread.join() - if self.playback_proxy is not None: - playback_proxy_thread.join() - - if self.service_registry and hasattr(self, "status_info"): - self.service_registry.unregister(self.status_info["id"]) + self.shutdown() diff --git a/warcprox/main.py b/warcprox/main.py index 00d2d85..b203c1c 100644 --- a/warcprox/main.py +++ b/warcprox/main.py @@ -209,14 +209,15 @@ def init_controller(args): warc_writer_thread, playback_proxy, service_registry=svcreg, options=options) - signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set()) - signal.signal(signal.SIGINT, lambda a,b: controller.stop.set()) - signal.signal(signal.SIGQUIT, dump_state) - return controller def real_main(args): controller = init_controller(args) + + signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set()) + signal.signal(signal.SIGINT, lambda a,b: controller.stop.set()) + signal.signal(signal.SIGQUIT, dump_state) + controller.run_until_shutdown() def parse_args(argv=sys.argv):