From 3b3e190cf45229e0b125ea7462173899839f9b0a Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 11 Mar 2016 11:10:22 -0800 Subject: [PATCH] testing: use test mixins for class-scope temp directory, live server creation use processes instead of threads for live server --- webagg/test/test_dir_agg.py | 6 ++-- webagg/test/test_upstream.py | 49 +++----------------------- webagg/test/testutils.py | 67 ++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 47 deletions(-) diff --git a/webagg/test/test_dir_agg.py b/webagg/test/test_dir_agg.py index 165b6346..b55d3755 100644 --- a/webagg/test/test_dir_agg.py +++ b/webagg/test/test_dir_agg.py @@ -3,7 +3,7 @@ import os import shutil import json -from .testutils import to_path, to_json_list, TempDirTests +from .testutils import to_path, to_json_list, TempDirTests, BaseTestClass from mock import patch @@ -21,7 +21,7 @@ def mock_link_header(*args, **kwargs): return linkheader -class TestDirAgg(TempDirTests): +class TestDirAgg(TempDirTests, BaseTestClass): @classmethod def setup_class(cls): super(TestDirAgg, cls).setup_class() @@ -40,7 +40,7 @@ class TestDirAgg(TempDirTests): shutil.copy(to_path('testdata/iana.cdxj'), coll_B) shutil.copy(to_path('testdata/dupes.cdxj'), coll_C) - with open(to_path(cls.root_dir) + 'somefile', 'w') as fh: + with open(to_path(cls.root_dir) + '/somefile', 'w') as fh: fh.write('foo') cls.dir_loader = DirectoryIndexSource(dir_prefix, dir_path) diff --git a/webagg/test/test_upstream.py b/webagg/test/test_upstream.py index 505b8edb..037b62e9 100644 --- a/webagg/test/test_upstream.py +++ b/webagg/test/test_upstream.py @@ -1,62 +1,23 @@ -from webagg.app import ResAggApp - import webtest -import threading from io import BytesIO +from webagg.app import ResAggApp import requests from webagg.handlers import DefaultResourceHandler -from webagg.indexsource import LiveIndexSource -from webagg.proxyindexsource import ProxyMementoIndexSource, UpstreamAggIndexSource from webagg.aggregator import SimpleAggregator - -from wsgiref.simple_server import make_server +from webagg.proxyindexsource import ProxyMementoIndexSource, UpstreamAggIndexSource from pywb.warc.recordloader import ArcWarcRecordLoader - -class ServerThreadRunner(object): - def __init__(self, app): - self.httpd = make_server('', 0, app) - self.port = self.httpd.socket.getsockname()[1] - - def run(): - self.httpd.serve_forever() - - self.thread = threading.Thread(target=run) - self.thread.daemon = True - self.thread.start() - - def stop_thread(self): - self.httpd.shutdown() +from .testutils import LiveServerTests, BaseTestClass -server = None - - -def setup_module(): - app = ResAggApp() - app.add_route('/live', - DefaultResourceHandler(SimpleAggregator( - {'live': LiveIndexSource()}) - ) - ) - - global server - server = ServerThreadRunner(app.application) - -def teardown_module(): - global server - server.stop_thread() - - - -class TestUpstream(object): +class TestUpstream(LiveServerTests, BaseTestClass): def setup(self): app = ResAggApp() - base_url = 'http://localhost:{0}'.format(server.port) + base_url = 'http://localhost:{0}'.format(self.server.port) app.add_route('/upstream', DefaultResourceHandler(SimpleAggregator( {'upstream': UpstreamAggIndexSource(base_url + '/live')}) diff --git a/webagg/test/testutils.py b/webagg/test/testutils.py index 61f8b155..51d91364 100644 --- a/webagg/test/testutils.py +++ b/webagg/test/testutils.py @@ -3,6 +3,17 @@ import os import tempfile import shutil +from multiprocessing import Process + +from wsgiref.simple_server import make_server + +from webagg.aggregator import SimpleAggregator +from webagg.app import ResAggApp +from webagg.handlers import DefaultResourceHandler +from webagg.indexsource import LiveIndexSource + + +# ============================================================================ def to_json_list(cdxlist, fields=['timestamp', 'load_url', 'filename', 'source']): return list([json.loads(cdx.to_json(fields)) for cdx in cdxlist]) @@ -16,12 +27,68 @@ def to_path(path): return path +# ============================================================================ +class BaseTestClass(object): + @classmethod + def setup_class(cls): + pass + + @classmethod + def teardown_class(cls): + pass + + +# ============================================================================ class TempDirTests(object): @classmethod def setup_class(cls): + super(TempDirTests, cls).setup_class() cls.root_dir = tempfile.mkdtemp() @classmethod def teardown_class(cls): + super(TempDirTests, cls).teardown_class() shutil.rmtree(cls.root_dir) + +# ============================================================================ +class LiveServerTests(object): + @classmethod + def setup_class(cls): + super(LiveServerTests, cls).setup_class() + cls.server = ServerThreadRunner(cls.make_live_app()) + + @staticmethod + def make_live_app(): + app = ResAggApp() + app.add_route('/live', + DefaultResourceHandler(SimpleAggregator( + {'live': LiveIndexSource()}) + ) + ) + return app.application + + @classmethod + def teardown_class(cls): + super(LiveServerTests, cls).teardown_class() + cls.server.stop_thread() + + +# ============================================================================ +class ServerThreadRunner(object): + def __init__(self, app): + self.httpd = make_server('', 0, app) + self.port = self.httpd.socket.getsockname()[1] + + def run(): + self.httpd.serve_forever() + + self.proc = Process(target=run) + #self.proc.daemon = True + self.proc.start() + + def stop_thread(self): + #self.httpd.shutdown() + self.proc.terminate() + +