reorganize WarcproxController.run_until_shutdown, moving parts of it into new start() and shutdown() methods, for easier integration into a separate python program

setup.py

@@ -1,37 +1,36 @@
 #!/usr/bin/env python
-#
-# setup.py - setuptools installation config for warcprox
-#
-# Copyright (C) 2013-2016 Internet Archive
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-#
+'''
+setup.py - setuptools installation configuration for warcprox
+
+Copyright (C) 2013-2016 Internet Archive
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+USA.
+'''
 
-from setuptools.command.test import test as TestCommand
 import sys
 import setuptools
 
 # special class needs to be added to support the pytest written dump-anydbm tests
-class PyTest(TestCommand):
+class PyTest(setuptools.command.test.TestCommand):
     def finalize_options(self):
-        TestCommand.finalize_options(self)
+        setuptools.command.test.TestCommand.finalize_options(self)
         self.test_args = []
         self.test_suite = True
     def run_tests(self):
-        #import here, cause outside the eggs aren't loaded
+        # import here, because outside the eggs aren't loaded
         import pytest
         errno = pytest.main(self.test_args)
         sys.exit(errno)
@@ -50,7 +49,7 @@ except:
     deps.append('futures')
 
 setuptools.setup(name='warcprox',
-        version='2.0.dev10',
+        version='2.0.dev11',
         description='WARC writing MITM HTTP/S proxy',
         url='https://github.com/internetarchive/warcprox',
         author='Noah Levitt',

warcprox/controller.py

@@ -146,27 +146,60 @@ class WarcproxController(object):
                 warcprox.TRACE, "status in service registry: %s",
                 self.status_info)
 
-    def run_until_shutdown(self):
-        """
-        Start warcprox and run until shut down. Call
-        warcprox_controller.stop.set() to initiate graceful shutdown.
-        """
+    def start(self):
+        # XXX check if already started
         if self.proxy.stats_db:
             self.proxy.stats_db.start()
-        proxy_thread = threading.Thread(
+        self.proxy_thread = threading.Thread(
                 target=self.proxy.serve_forever, name='ProxyThread')
-        proxy_thread.start()
+        self.proxy_thread.start()
 
         if self.warc_writer_thread.dedup_db:
             self.warc_writer_thread.dedup_db.start()
         self.warc_writer_thread.start()
 
         if self.playback_proxy is not None:
-            playback_proxy_thread = threading.Thread(target=self.playback_proxy.serve_forever, name='PlaybackProxyThread')
+            self.playback_proxy_thread = threading.Thread(
+                    target=self.playback_proxy.serve_forever,
+                    name='PlaybackProxyThread')
             playback_proxy_thread.start()
 
         self.stop = threading.Event()
 
+    def shutdown(self):
+        # XXX check if already shut down
+        self.warc_writer_thread.stop.set()
+        self.proxy.shutdown()
+        self.proxy.server_close()
+
+        if self.playback_proxy is not None:
+            self.playback_proxy.shutdown()
+            self.playback_proxy.server_close()
+            if self.playback_proxy.playback_index_db is not None:
+                self.playback_proxy.playback_index_db.close()
+
+        # wait for threads to finish
+        self.warc_writer_thread.join()
+
+        if self.proxy.stats_db:
+            self.proxy.stats_db.stop()
+        if self.warc_writer_thread.dedup_db:
+            self.warc_writer_thread.dedup_db.close()
+
+        self.proxy_thread.join()
+        if self.playback_proxy is not None:
+            self.playback_proxy_thread.join()
+
+        if self.service_registry and hasattr(self, "status_info"):
+            self.service_registry.unregister(self.status_info["id"])
+
+    def run_until_shutdown(self):
+        """
+        Start warcprox and run until shut down. Call
+        warcprox_controller.stop.set() to initiate graceful shutdown.
+        """
+        self.start()
+
         last_mem_dbg = datetime.datetime.utcfromtimestamp(0)
 
         try:
@@ -190,31 +223,10 @@ class WarcproxController(object):
 
                 time.sleep(0.5)
         except:
-            self.logger.critical("fatal exception, shutting down", exc_info=True)
+            self.logger.critical(
+                    "shutting down in response to fatal exception",
+                    exc_info=True)
             pass
         finally:
-            self.warc_writer_thread.stop.set()
-            self.proxy.shutdown()
-            self.proxy.server_close()
-
-            if self.playback_proxy is not None:
-                self.playback_proxy.shutdown()
-                self.playback_proxy.server_close()
-                if self.playback_proxy.playback_index_db is not None:
-                    self.playback_proxy.playback_index_db.close()
-
-            # wait for threads to finish
-            self.warc_writer_thread.join()
-
-            if self.proxy.stats_db:
-                self.proxy.stats_db.stop()
-            if self.warc_writer_thread.dedup_db:
-                self.warc_writer_thread.dedup_db.close()
-
-            proxy_thread.join()
-            if self.playback_proxy is not None:
-                playback_proxy_thread.join()
-
-            if self.service_registry and hasattr(self, "status_info"):
-                self.service_registry.unregister(self.status_info["id"])
+            self.shutdown()
 

warcprox/main.py

@@ -209,14 +209,15 @@ def init_controller(args):
         warc_writer_thread, playback_proxy, service_registry=svcreg,
         options=options)
 
-    signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set())
-    signal.signal(signal.SIGINT, lambda a,b: controller.stop.set())
-    signal.signal(signal.SIGQUIT, dump_state)
-
     return controller
 
 def real_main(args):
     controller = init_controller(args)
+
+    signal.signal(signal.SIGTERM, lambda a,b: controller.stop.set())
+    signal.signal(signal.SIGINT, lambda a,b: controller.stop.set())
+    signal.signal(signal.SIGQUIT, dump_state)
+
     controller.run_until_shutdown()
 
 def parse_args(argv=sys.argv):