apps & cli: remove old apps, keep:

- webagg-server
- wayback
- live-rewrite-server
support adding custom settings to AutoApp
support for --live flag that automatically adds live-web source at '/live'
tests: disable cdx_server tests as old cdx_server removed

pywb/apps/cdx_server.py

@@ -1,11 +0,0 @@
-from pywb.framework.wsgi_wrappers import init_app
-
-#from pywb.core.cdx_api_handler import create_cdx_server_app
-from pywb.webapp.pywb_init import create_cdx_server_app
-
-#=================================================================
-# init cdx server app
-#=================================================================
-
-application = init_app(create_cdx_server_app,
-                       load_yaml=True)

pywb/apps/cli.py

@@ -1,36 +1,28 @@
+from gevent.monkey import patch_all; patch_all()
 from argparse import ArgumentParser
+
 import logging
 
 
-#=================================================================
+#=============================================================================
-def cdx_server(args=None):  #pragma: no cover
+def webagg(args=None):
-    CdxCli(args=args,
+    WebaggCli(args=args,
-           default_port=8080,
+              default_port=8070,
-           desc='pywb CDX Index Server').run()
+              desc='pywb Web Aggregator Server').run()
 
 
-#=================================================================
+#=============================================================================
-def live_rewrite_server(args=None):  #pragma: no cover
-    NewLiveCli(args=args,
-            default_port=8090,
-            desc='pywb Live Rewrite Proxy Server').run()
-
-
-#=================================================================
 def wayback(args=None):
     WaybackCli(args=args,
                default_port=8080,
-               desc='pywb Wayback Web Archive Replay').run()
+               desc='pywb Wayback Machine Server').run()
 
 
 #=============================================================================
-def webagg():
+def live_rewrite_server(args=None):
-    WebaggCli().run()
+    LiveCli(args=args,
-
+            default_port=8090,
-
+            desc='pywb Live Rewrite Proxy Server').run()
-#=============================================================================
-def new_wayback():
-    NewWaybackCli().run()
 
 
 #=============================================================================
@@ -39,11 +31,13 @@ class BaseCli(object):
         parser = ArgumentParser(description=desc)
         parser.add_argument('-p', '--port', type=int, default=default_port)
         parser.add_argument('-t', '--threads', type=int, default=4)
-        parser.add_argument('-s', '--server', default='gevent')
         parser.add_argument('--debug', action='store_true')
         parser.add_argument('--profile', action='store_true')
 
+        parser.add_argument('--live', action='store_true', help='Add /live handler')
+
         self.desc = desc
+        self.extra_config = None
 
         self._extend_parser(parser)
 
@@ -52,14 +46,6 @@ class BaseCli(object):
         logging.basicConfig(format='%(asctime)s: [%(levelname)s]: %(message)s',
                             level=logging.DEBUG if self.r.debug else logging.INFO)
 
-        if self.r.server == 'gevent':
-            try:
-                from gevent.monkey import patch_all; patch_all()
-                logging.debug('Using Gevent')
-            except:
-                logging.debug('No Gevent')
-                self.r.server = 'wsgiref'
-
         self.application = self.load()
 
         if self.r.profile:
@@ -69,25 +55,12 @@ class BaseCli(object):
     def _extend_parser(self, parser):  #pragma: no cover
         pass
 
-    def load(self):  #pragma: no cover
+    def load(self):
-        pass
+        if self.r.live:
+            self.extra_config = {'collections': {'live': '$live'}}
 
     def run(self):
-        if self.r.server == 'gevent':  #pragma: no cover
         self.run_gevent()
-        elif self.r.server == 'waitress':  #pragma: no cover
-            self.run_waitress()
-        else:
-            self.run_wsgiref()
-
-    def run_waitress(self):  #pragma: no cover
-        from waitress import serve
-        logging.debug(str(self.desc))
-        serve(self.application, port=self.r.port, threads=self.r.threads)
-
-    def run_wsgiref(self):  #pragma: no cover
-        from pywb.framework.wsgi_wrappers import start_wsgi_ref_server
-        start_wsgi_ref_server(self.application, self.desc, port=self.r.port)
 
     def run_gevent(self):
         from gevent.pywsgi import WSGIServer
@@ -95,27 +68,6 @@ class BaseCli(object):
         WSGIServer(('', self.r.port), self.application).serve_forever()
 
 
-#=============================================================================
-class LiveCli(BaseCli):
-    def _extend_parser(self, parser):
-        parser.add_argument('-x', '--proxy',
-                            help='Specify host:port to use as HTTP/S proxy')
-
-        parser.add_argument('-f', '--framed', action='store_true',
-                            help='Replay using framed wrapping mode')
-
-    def load(self):
-        config = dict(proxyhostport=self.r.proxy,
-                      framed_replay='inverse' if self.r.framed else False,
-                      enable_auto_colls=False,
-                      collections={'live': '$liveweb'})
-
-        from pywb.webapp.pywb_init import create_wb_router
-        from pywb.framework.wsgi_wrappers import init_app
-
-        return init_app(create_wb_router, load_yaml=False, config=config)
-
-
 #=============================================================================
 class ReplayCli(BaseCli):
     def _extend_parser(self, parser):
@@ -127,6 +79,7 @@ class ReplayCli(BaseCli):
 
 
     def load(self):
+        super(ReplayCli, self).load()
         if self.r.directory:  #pragma: no cover
             os.chdir(self.r.directory)
 
@@ -150,58 +103,32 @@ class ReplayCli(BaseCli):
 
 
 #=============================================================================
-class CdxCli(ReplayCli):  #pragma: no cover
+class WebaggCli(BaseCli):
     def load(self):
-        from pywb.webapp.pywb_init import create_cdx_server_app
+        from pywb.webagg.autoapp import AutoConfigApp
-        from pywb.framework.wsgi_wrappers import init_app
+
-        super(CdxCli, self).load()
+        super(WebaggCli, self).load()
-        return init_app(create_cdx_server_app,
+        return AutoConfigApp(custom_config=self.extra_config)
-                        load_yaml=True)
 
 
 #=============================================================================
 class WaybackCli(ReplayCli):
     def load(self):
-        from pywb.webapp.pywb_init import create_wb_router
+        from pywb.urlrewrite.frontendapp import FrontEndApp
-        from pywb.framework.wsgi_wrappers import init_app
+
         super(WaybackCli, self).load()
-        return init_app(create_wb_router,
+        return FrontEndApp(custom_config=self.extra_config)
-                        load_yaml=True)
 
 
 #=============================================================================
-class WebaggCli(BaseCli):
+class LiveCli(BaseCli):
     def load(self):
-        from pywb.apps.webagg import application
+        from pywb.urlrewrite.frontendapp import FrontEndApp
-        return application
-
-    def run(self):
-        self.run_gevent()
-
-
-#=============================================================================
-class NewWaybackCli(ReplayCli):
-    def load(self):
-        from pywb.apps.newwayback import application
-        return application
-
-    def run(self):
-        self.r.server = 'gevent'
-        super(NewWaybackCli, self).run()
-        #self.run_gevent()
-
-#=============================================================================
-class NewLiveCli(BaseCli):
-    def load(self):
-        from pywb.apps.live import application
-        return application
-
-    def run(self):
-        self.r.server = 'gevent'
-        super(NewLiveCli, self).run()
-        #self.run_gevent()
 
+        self.r.live = True
 
+        super(LiveCli, self).load()
+        return FrontEndApp(config_file=None, custom_config=self.extra_config)
 
 
 #=============================================================================

pywb/apps/newwayback.py

@@ -1,6 +0,0 @@
-from gevent.monkey import patch_all; patch_all()
-from pywb.urlrewrite.frontendapp import FrontEndApp
-
-application = FrontEndApp()
-
-

pywb/apps/wayback.py

@@ -1,18 +1,6 @@
-import os
+from gevent.monkey import patch_all; patch_all()
+from pywb.urlrewrite.frontendapp import FrontEndApp
 
-if os.environ.get('GEVENT_MONKEY_PATCH') == '1':  #pragma: no cover
+application = FrontEndApp()
-    # Use gevent if available
-    try:
-        from gevent.monkey import patch_all; patch_all()
-        print('gevent patched!')
-    except Exception as e:
-        pass
-
-from pywb.framework.wsgi_wrappers import init_app
-from pywb.webapp.pywb_init import create_wb_router
 
 
-#=================================================================
-# init pywb app
-#=================================================================
-application = init_app(create_wb_router, load_yaml=True)

pywb/apps/webagg.py

@@ -1,6 +1,7 @@
 from gevent.monkey import patch_all; patch_all()
 from pywb.webagg.autoapp import AutoConfigApp
 
-application = AutoConfigApp()
+application = AutoConfigApp(custom_config={'collections': {'live': '$live'}})
+
 
 

pywb/webagg/autoapp.py

@@ -39,13 +39,16 @@ class AutoConfigApp(ResAggApp):
 
         if config_file:
             try:
-                custom_config = load_config('PYWB_CONFIG_FILE', config_file)
+                file_config = load_config('PYWB_CONFIG_FILE', config_file)
+                config.update(file_config)
             except Exception as e:
                 if not custom_config:
                     custom_config = {'debug': True}
                 print(e)
 
         if custom_config:
+            if 'collections' in custom_config and 'collections' in config:
+                custom_config['collections'].update(config['collections'])
             config.update(custom_config)
 
         super(AutoConfigApp, self).__init__(debug=config.get('debug', False))

tests/test_auto_colls.py

@@ -87,17 +87,17 @@ class TestManagedColls(object):
         """
         from pywb.apps.cli import wayback
 
-        wayback(['-p', '0', '-s', 'gevent'])
+        wayback(['-p', '0'])
 
         # Nothing to auto-index.. yet
         with raises(SystemExit):
-            wayback(['-a', '-p', '0', '-s', 'gevent'])
+            wayback(['-a', '-p', '0'])
 
         colls = os.path.join(self.root_dir, 'collections')
         os.mkdir(colls)
 
         pywb.manager.autoindex.keep_running = False
-        wayback(['-a', '-p', '0', '-s', 'gevent'])
+        wayback(['-a', '-p', '0'])
 
     def test_create_first_coll(self):
         """ Test first collection creation, with all required dirs