wsgi server: default to gevent-based wsgi server for all cmd line server apps, add -s command for specifying server #201

cli: add 'webagg-server' cli command for running new webagg system
tests: fix cli test for gevent server

pywb/apps/cli.py

@@ -1,9 +1,4 @@
-import os
-import logging
 from argparse import ArgumentParser
-from pywb.framework.wsgi_wrappers import init_app
-from pywb.webapp.pywb_init import create_cdx_server_app, create_wb_router
-
 
 #=================================================================
 def cdx_server(args=None):  #pragma: no cover
@@ -26,13 +21,18 @@ def wayback(args=None):
                desc='pywb Wayback Web Archive Replay').run()
 
 
+#=============================================================================
+def webagg():
+    WebaggCli().run()
+
+
 #=============================================================================
 class BaseCli(object):
     def __init__(self, args=None, default_port=8080, desc=''):
         parser = ArgumentParser(description=desc)
         parser.add_argument('-p', '--port', type=int, default=default_port)
         parser.add_argument('-t', '--threads', type=int, default=4)
-        parser.add_argument('-s', '--server')
+        parser.add_argument('-s', '--server', default='gevent')
 
         self.desc = desc
 
@@ -40,6 +40,17 @@ class BaseCli(object):
 
         self.r = parser.parse_args(args)
 
+        if self.r.server == 'gevent':
+            try:
+                from gevent.monkey import patch_all; patch_all()
+                print('Using Gevent')
+            except:
+                print('No Gevent')
+                self.r.server = 'wsgiref'
+
+        from pywb.framework.wsgi_wrappers import init_app
+        self.init_app = init_app
+
         self.application = self.load()
 
     def _extend_parser(self, parser):  #pragma: no cover
@@ -49,7 +60,9 @@ class BaseCli(object):
         pass
 
     def run(self):
-        if self.r.server == 'waitress':  #pragma: no cover
+        if self.r.server == 'gevent':  #pragma: no cover
+            self.run_gevent()
+        elif self.r.server == 'waitress':  #pragma: no cover
             self.run_waitress()
         else:
             self.run_wsgiref()
@@ -63,6 +76,11 @@ class BaseCli(object):
         from pywb.framework.wsgi_wrappers import start_wsgi_ref_server
         start_wsgi_ref_server(self.application, self.desc, port=self.r.port)
 
+    def run_gevent(self):
+        from gevent.pywsgi import WSGIServer
+        print('Starting Gevent Server on ' + str(self.r.port))
+        WSGIServer(('', self.r.port), self.application).serve_forever()
+
 
 #=============================================================================
 class LiveCli(BaseCli):
@@ -79,7 +97,8 @@ class LiveCli(BaseCli):
                       enable_auto_colls=False,
                       collections={'live': '$liveweb'})
 
-        return init_app(create_wb_router, load_yaml=False, config=config)
+        from pywb.webapp.pywb_init import create_wb_router
+        return self.init_app(create_wb_router, load_yaml=False, config=config)
 
 
 #=============================================================================
@@ -98,6 +117,9 @@ class ReplayCli(BaseCli):
     def run(self):
         if self.r.autoindex:
             from pywb.manager.manager import CollectionsManager
+            import os
+            import logging
+
             m = CollectionsManager('', must_exist=False)
             if not os.path.isdir(m.colls_dir):
                 msg = 'No managed directory "{0}" for auto-indexing'
@@ -114,17 +136,29 @@ class ReplayCli(BaseCli):
 #=============================================================================
 class CdxCli(ReplayCli):  #pragma: no cover
     def load(self):
+        from pywb.webapp.pywb_init import create_cdx_server_app
         super(CdxCli, self).load()
-        return init_app(create_cdx_server_app,
+        return self.init_app(create_cdx_server_app,
-                        load_yaml=True)
+                             load_yaml=True)
 
 
 #=============================================================================
 class WaybackCli(ReplayCli):
     def load(self):
+        from pywb.webapp.pywb_init import create_wb_router
         super(WaybackCli, self).load()
-        return init_app(create_wb_router,
+        return self.init_app(create_wb_router,
-                        load_yaml=True)
+                             load_yaml=True)
+
+
+#=============================================================================
+class WebaggCli(BaseCli):
+    def load(self):
+        from pywb.apps.webagg import application
+        return application
+
+    def run(self):
+        self.run_gevent()
 
 
 #=============================================================================

setup.py

@@ -107,6 +107,7 @@ setup(
         live-rewrite-server = pywb.apps.cli:live_rewrite_server
         cdx-indexer = pywb.warc.cdxindexer:main
         wb-manager = pywb.manager.manager:main_wrap_exc
+        webagg-server = pywb.apps.cli:webagg
         """,
     classifiers=[
         'Development Status :: 4 - Beta',

tests/test_auto_colls.py

@@ -80,25 +80,24 @@ class TestManagedColls(object):
     def teardown(self):
         J2TemplateView.shared_jinja_env = None
 
-    #@patch('waitress.serve', lambda *args, **kwargs: None)
+    @patch('pywb.apps.cli.BaseCli.run_gevent', lambda *args, **kwargs: None)
-    @patch('six.moves.BaseHTTPServer.HTTPServer.serve_forever', lambda *args, **kwargs: None)
     def test_run_cli(self):
         """ test new wayback cli interface
         test autoindex error before collections inited
         """
         from pywb.apps.cli import wayback
 
-        wayback(['-p', '0'])
+        wayback(['-p', '0', '-s', 'gevent'])
 
         # Nothing to auto-index.. yet
         with raises(SystemExit):
-            wayback(['-a', '-p', '0'])
+            wayback(['-a', '-p', '0', '-s', 'gevent'])
 
         colls = os.path.join(self.root_dir, 'collections')
         os.mkdir(colls)
 
         pywb.manager.autoindex.keep_running = False
-        wayback(['-a', '-p', '0'])
+        wayback(['-a', '-p', '0', '-s', 'gevent'])
 
     def test_create_first_coll(self):
         """ Test first collection creation, with all required dirs