testing: use test mixins for class-scope temp directory, live server creation

use processes instead of threads for live server

webagg/test/test_dir_agg.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import json
 
-from .testutils import to_path, to_json_list, TempDirTests
+from .testutils import to_path, to_json_list, TempDirTests, BaseTestClass
 
 from mock import patch
 
@@ -21,7 +21,7 @@ def mock_link_header(*args, **kwargs):
     return linkheader
 
 
-class TestDirAgg(TempDirTests):
+class TestDirAgg(TempDirTests, BaseTestClass):
     @classmethod
     def setup_class(cls):
         super(TestDirAgg, cls).setup_class()
@@ -40,7 +40,7 @@ class TestDirAgg(TempDirTests):
         shutil.copy(to_path('testdata/iana.cdxj'), coll_B)
         shutil.copy(to_path('testdata/dupes.cdxj'), coll_C)
 
-        with open(to_path(cls.root_dir) + 'somefile', 'w') as fh:
+        with open(to_path(cls.root_dir) + '/somefile', 'w') as fh:
             fh.write('foo')
 
         cls.dir_loader = DirectoryIndexSource(dir_prefix, dir_path)

webagg/test/test_upstream.py

@@ -1,62 +1,23 @@
-from webagg.app import ResAggApp
-
 import webtest
-import threading
 
 from io import BytesIO
+from webagg.app import ResAggApp
 import requests
 
 from webagg.handlers import DefaultResourceHandler
-from webagg.indexsource import LiveIndexSource
-from webagg.proxyindexsource import ProxyMementoIndexSource, UpstreamAggIndexSource
 from webagg.aggregator import SimpleAggregator
-
-from wsgiref.simple_server import make_server
+from webagg.proxyindexsource import ProxyMementoIndexSource, UpstreamAggIndexSource
 
 from pywb.warc.recordloader import ArcWarcRecordLoader
 
-
-class ServerThreadRunner(object):
-    def __init__(self, app):
-        self.httpd = make_server('', 0, app)
-        self.port = self.httpd.socket.getsockname()[1]
-
-        def run():
-            self.httpd.serve_forever()
-
-        self.thread = threading.Thread(target=run)
-        self.thread.daemon = True
-        self.thread.start()
-
-    def stop_thread(self):
-        self.httpd.shutdown()
+from .testutils import LiveServerTests, BaseTestClass
 
 
-server = None
-
-
-def setup_module():
-    app = ResAggApp()
-    app.add_route('/live',
-        DefaultResourceHandler(SimpleAggregator(
-                               {'live': LiveIndexSource()})
-        )
-    )
-
-    global server
-    server = ServerThreadRunner(app.application)
-
-def teardown_module():
-    global server
-    server.stop_thread()
-
-
-
-class TestUpstream(object):
+class TestUpstream(LiveServerTests, BaseTestClass):
     def setup(self):
         app = ResAggApp()
 
-        base_url = 'http://localhost:{0}'.format(server.port)
+        base_url = 'http://localhost:{0}'.format(self.server.port)
         app.add_route('/upstream',
             DefaultResourceHandler(SimpleAggregator(
                            {'upstream': UpstreamAggIndexSource(base_url + '/live')})

webagg/test/testutils.py

@@ -3,6 +3,17 @@ import os
 import tempfile
 import shutil
 
+from multiprocessing import Process
+
+from wsgiref.simple_server import make_server
+
+from webagg.aggregator import SimpleAggregator
+from webagg.app import ResAggApp
+from webagg.handlers import DefaultResourceHandler
+from webagg.indexsource import LiveIndexSource
+
+
+# ============================================================================
 def to_json_list(cdxlist, fields=['timestamp', 'load_url', 'filename', 'source']):
     return list([json.loads(cdx.to_json(fields)) for cdx in cdxlist])
 
@@ -16,12 +27,68 @@ def to_path(path):
     return path
 
 
+# ============================================================================
+class BaseTestClass(object):
+    @classmethod
+    def setup_class(cls):
+        pass
+
+    @classmethod
+    def teardown_class(cls):
+        pass
+
+
+# ============================================================================
 class TempDirTests(object):
     @classmethod
     def setup_class(cls):
+        super(TempDirTests, cls).setup_class()
         cls.root_dir = tempfile.mkdtemp()
 
     @classmethod
     def teardown_class(cls):
+        super(TempDirTests, cls).teardown_class()
         shutil.rmtree(cls.root_dir)
 
+
+# ============================================================================
+class LiveServerTests(object):
+    @classmethod
+    def setup_class(cls):
+        super(LiveServerTests, cls).setup_class()
+        cls.server = ServerThreadRunner(cls.make_live_app())
+
+    @staticmethod
+    def make_live_app():
+        app = ResAggApp()
+        app.add_route('/live',
+            DefaultResourceHandler(SimpleAggregator(
+                                   {'live': LiveIndexSource()})
+            )
+        )
+        return app.application
+
+    @classmethod
+    def teardown_class(cls):
+        super(LiveServerTests, cls).teardown_class()
+        cls.server.stop_thread()
+
+
+# ============================================================================
+class ServerThreadRunner(object):
+    def __init__(self, app):
+        self.httpd = make_server('', 0, app)
+        self.port = self.httpd.socket.getsockname()[1]
+
+        def run():
+            self.httpd.serve_forever()
+
+        self.proc = Process(target=run)
+        #self.proc.daemon = True
+        self.proc.start()
+
+    def stop_thread(self):
+        #self.httpd.shutdown()
+        self.proc.terminate()
+
+