From 50a3353da389cc7ffffa2365265b86c2d4892d2c Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 9 Dec 2016 16:46:33 -0800 Subject: [PATCH] wsgi server: default to gevent-based wsgi server for all cmd line server apps, add -s command for specifying server #201 cli: add 'webagg-server' cli command for running new webagg system tests: fix cli test for gevent server --- pywb/apps/cli.py | 58 +++++++++++++++++++++++++++++++--------- setup.py | 1 + tests/test_auto_colls.py | 9 +++---- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/pywb/apps/cli.py b/pywb/apps/cli.py index 0d54374a..fe404dbd 100644 --- a/pywb/apps/cli.py +++ b/pywb/apps/cli.py @@ -1,9 +1,4 @@ -import os -import logging from argparse import ArgumentParser -from pywb.framework.wsgi_wrappers import init_app -from pywb.webapp.pywb_init import create_cdx_server_app, create_wb_router - #================================================================= def cdx_server(args=None): #pragma: no cover @@ -26,13 +21,18 @@ def wayback(args=None): desc='pywb Wayback Web Archive Replay').run() +#============================================================================= +def webagg(): + WebaggCli().run() + + #============================================================================= class BaseCli(object): def __init__(self, args=None, default_port=8080, desc=''): parser = ArgumentParser(description=desc) parser.add_argument('-p', '--port', type=int, default=default_port) parser.add_argument('-t', '--threads', type=int, default=4) - parser.add_argument('-s', '--server') + parser.add_argument('-s', '--server', default='gevent') self.desc = desc @@ -40,6 +40,17 @@ class BaseCli(object): self.r = parser.parse_args(args) + if self.r.server == 'gevent': + try: + from gevent.monkey import patch_all; patch_all() + print('Using Gevent') + except: + print('No Gevent') + self.r.server = 'wsgiref' + + from pywb.framework.wsgi_wrappers import init_app + self.init_app = init_app + self.application = self.load() def _extend_parser(self, parser): #pragma: no cover @@ -49,7 +60,9 @@ class BaseCli(object): pass def run(self): - if self.r.server == 'waitress': #pragma: no cover + if self.r.server == 'gevent': #pragma: no cover + self.run_gevent() + elif self.r.server == 'waitress': #pragma: no cover self.run_waitress() else: self.run_wsgiref() @@ -63,6 +76,11 @@ class BaseCli(object): from pywb.framework.wsgi_wrappers import start_wsgi_ref_server start_wsgi_ref_server(self.application, self.desc, port=self.r.port) + def run_gevent(self): + from gevent.pywsgi import WSGIServer + print('Starting Gevent Server on ' + str(self.r.port)) + WSGIServer(('', self.r.port), self.application).serve_forever() + #============================================================================= class LiveCli(BaseCli): @@ -79,7 +97,8 @@ class LiveCli(BaseCli): enable_auto_colls=False, collections={'live': '$liveweb'}) - return init_app(create_wb_router, load_yaml=False, config=config) + from pywb.webapp.pywb_init import create_wb_router + return self.init_app(create_wb_router, load_yaml=False, config=config) #============================================================================= @@ -98,6 +117,9 @@ class ReplayCli(BaseCli): def run(self): if self.r.autoindex: from pywb.manager.manager import CollectionsManager + import os + import logging + m = CollectionsManager('', must_exist=False) if not os.path.isdir(m.colls_dir): msg = 'No managed directory "{0}" for auto-indexing' @@ -114,17 +136,29 @@ class ReplayCli(BaseCli): #============================================================================= class CdxCli(ReplayCli): #pragma: no cover def load(self): + from pywb.webapp.pywb_init import create_cdx_server_app super(CdxCli, self).load() - return init_app(create_cdx_server_app, - load_yaml=True) + return self.init_app(create_cdx_server_app, + load_yaml=True) #============================================================================= class WaybackCli(ReplayCli): def load(self): + from pywb.webapp.pywb_init import create_wb_router super(WaybackCli, self).load() - return init_app(create_wb_router, - load_yaml=True) + return self.init_app(create_wb_router, + load_yaml=True) + + +#============================================================================= +class WebaggCli(BaseCli): + def load(self): + from pywb.apps.webagg import application + return application + + def run(self): + self.run_gevent() #============================================================================= diff --git a/setup.py b/setup.py index afe7aa2e..40cce73c 100755 --- a/setup.py +++ b/setup.py @@ -107,6 +107,7 @@ setup( live-rewrite-server = pywb.apps.cli:live_rewrite_server cdx-indexer = pywb.warc.cdxindexer:main wb-manager = pywb.manager.manager:main_wrap_exc + webagg-server = pywb.apps.cli:webagg """, classifiers=[ 'Development Status :: 4 - Beta', diff --git a/tests/test_auto_colls.py b/tests/test_auto_colls.py index ff88d947..6e257432 100644 --- a/tests/test_auto_colls.py +++ b/tests/test_auto_colls.py @@ -80,25 +80,24 @@ class TestManagedColls(object): def teardown(self): J2TemplateView.shared_jinja_env = None - #@patch('waitress.serve', lambda *args, **kwargs: None) - @patch('six.moves.BaseHTTPServer.HTTPServer.serve_forever', lambda *args, **kwargs: None) + @patch('pywb.apps.cli.BaseCli.run_gevent', lambda *args, **kwargs: None) def test_run_cli(self): """ test new wayback cli interface test autoindex error before collections inited """ from pywb.apps.cli import wayback - wayback(['-p', '0']) + wayback(['-p', '0', '-s', 'gevent']) # Nothing to auto-index.. yet with raises(SystemExit): - wayback(['-a', '-p', '0']) + wayback(['-a', '-p', '0', '-s', 'gevent']) colls = os.path.join(self.root_dir, 'collections') os.mkdir(colls) pywb.manager.autoindex.keep_running = False - wayback(['-a', '-p', '0']) + wayback(['-a', '-p', '0', '-s', 'gevent']) def test_create_first_coll(self): """ Test first collection creation, with all required dirs