diff --git a/pywb/apps/cdx_server.py b/pywb/apps/cdx_server.py deleted file mode 100644 index 83f49cd0..00000000 --- a/pywb/apps/cdx_server.py +++ /dev/null @@ -1,11 +0,0 @@ -from pywb.framework.wsgi_wrappers import init_app - -#from pywb.core.cdx_api_handler import create_cdx_server_app -from pywb.webapp.pywb_init import create_cdx_server_app - -#================================================================= -# init cdx server app -#================================================================= - -application = init_app(create_cdx_server_app, - load_yaml=True) diff --git a/pywb/apps/cli.py b/pywb/apps/cli.py index 548fe170..952451ef 100644 --- a/pywb/apps/cli.py +++ b/pywb/apps/cli.py @@ -1,36 +1,28 @@ +from gevent.monkey import patch_all; patch_all() from argparse import ArgumentParser + import logging -#================================================================= -def cdx_server(args=None): #pragma: no cover - CdxCli(args=args, - default_port=8080, - desc='pywb CDX Index Server').run() +#============================================================================= +def webagg(args=None): + WebaggCli(args=args, + default_port=8070, + desc='pywb Web Aggregator Server').run() -#================================================================= -def live_rewrite_server(args=None): #pragma: no cover - NewLiveCli(args=args, - default_port=8090, - desc='pywb Live Rewrite Proxy Server').run() - - -#================================================================= +#============================================================================= def wayback(args=None): WaybackCli(args=args, default_port=8080, - desc='pywb Wayback Web Archive Replay').run() + desc='pywb Wayback Machine Server').run() #============================================================================= -def webagg(): - WebaggCli().run() - - -#============================================================================= -def new_wayback(): - NewWaybackCli().run() +def live_rewrite_server(args=None): + LiveCli(args=args, + default_port=8090, + desc='pywb Live Rewrite Proxy Server').run() #============================================================================= @@ -39,11 +31,13 @@ class BaseCli(object): parser = ArgumentParser(description=desc) parser.add_argument('-p', '--port', type=int, default=default_port) parser.add_argument('-t', '--threads', type=int, default=4) - parser.add_argument('-s', '--server', default='gevent') parser.add_argument('--debug', action='store_true') parser.add_argument('--profile', action='store_true') + parser.add_argument('--live', action='store_true', help='Add /live handler') + self.desc = desc + self.extra_config = None self._extend_parser(parser) @@ -52,14 +46,6 @@ class BaseCli(object): logging.basicConfig(format='%(asctime)s: [%(levelname)s]: %(message)s', level=logging.DEBUG if self.r.debug else logging.INFO) - if self.r.server == 'gevent': - try: - from gevent.monkey import patch_all; patch_all() - logging.debug('Using Gevent') - except: - logging.debug('No Gevent') - self.r.server = 'wsgiref' - self.application = self.load() if self.r.profile: @@ -69,25 +55,12 @@ class BaseCli(object): def _extend_parser(self, parser): #pragma: no cover pass - def load(self): #pragma: no cover - pass + def load(self): + if self.r.live: + self.extra_config = {'collections': {'live': '$live'}} def run(self): - if self.r.server == 'gevent': #pragma: no cover - self.run_gevent() - elif self.r.server == 'waitress': #pragma: no cover - self.run_waitress() - else: - self.run_wsgiref() - - def run_waitress(self): #pragma: no cover - from waitress import serve - logging.debug(str(self.desc)) - serve(self.application, port=self.r.port, threads=self.r.threads) - - def run_wsgiref(self): #pragma: no cover - from pywb.framework.wsgi_wrappers import start_wsgi_ref_server - start_wsgi_ref_server(self.application, self.desc, port=self.r.port) + self.run_gevent() def run_gevent(self): from gevent.pywsgi import WSGIServer @@ -95,27 +68,6 @@ class BaseCli(object): WSGIServer(('', self.r.port), self.application).serve_forever() -#============================================================================= -class LiveCli(BaseCli): - def _extend_parser(self, parser): - parser.add_argument('-x', '--proxy', - help='Specify host:port to use as HTTP/S proxy') - - parser.add_argument('-f', '--framed', action='store_true', - help='Replay using framed wrapping mode') - - def load(self): - config = dict(proxyhostport=self.r.proxy, - framed_replay='inverse' if self.r.framed else False, - enable_auto_colls=False, - collections={'live': '$liveweb'}) - - from pywb.webapp.pywb_init import create_wb_router - from pywb.framework.wsgi_wrappers import init_app - - return init_app(create_wb_router, load_yaml=False, config=config) - - #============================================================================= class ReplayCli(BaseCli): def _extend_parser(self, parser): @@ -127,6 +79,7 @@ class ReplayCli(BaseCli): def load(self): + super(ReplayCli, self).load() if self.r.directory: #pragma: no cover os.chdir(self.r.directory) @@ -150,58 +103,32 @@ class ReplayCli(BaseCli): #============================================================================= -class CdxCli(ReplayCli): #pragma: no cover +class WebaggCli(BaseCli): def load(self): - from pywb.webapp.pywb_init import create_cdx_server_app - from pywb.framework.wsgi_wrappers import init_app - super(CdxCli, self).load() - return init_app(create_cdx_server_app, - load_yaml=True) + from pywb.webagg.autoapp import AutoConfigApp + + super(WebaggCli, self).load() + return AutoConfigApp(custom_config=self.extra_config) #============================================================================= class WaybackCli(ReplayCli): def load(self): - from pywb.webapp.pywb_init import create_wb_router - from pywb.framework.wsgi_wrappers import init_app + from pywb.urlrewrite.frontendapp import FrontEndApp + super(WaybackCli, self).load() - return init_app(create_wb_router, - load_yaml=True) + return FrontEndApp(custom_config=self.extra_config) #============================================================================= -class WebaggCli(BaseCli): +class LiveCli(BaseCli): def load(self): - from pywb.apps.webagg import application - return application - - def run(self): - self.run_gevent() - - -#============================================================================= -class NewWaybackCli(ReplayCli): - def load(self): - from pywb.apps.newwayback import application - return application - - def run(self): - self.r.server = 'gevent' - super(NewWaybackCli, self).run() - #self.run_gevent() - -#============================================================================= -class NewLiveCli(BaseCli): - def load(self): - from pywb.apps.live import application - return application - - def run(self): - self.r.server = 'gevent' - super(NewLiveCli, self).run() - #self.run_gevent() + from pywb.urlrewrite.frontendapp import FrontEndApp + self.r.live = True + super(LiveCli, self).load() + return FrontEndApp(config_file=None, custom_config=self.extra_config) #============================================================================= diff --git a/pywb/apps/newwayback.py b/pywb/apps/newwayback.py deleted file mode 100644 index 2febc1f4..00000000 --- a/pywb/apps/newwayback.py +++ /dev/null @@ -1,6 +0,0 @@ -from gevent.monkey import patch_all; patch_all() -from pywb.urlrewrite.frontendapp import FrontEndApp - -application = FrontEndApp() - - diff --git a/pywb/apps/wayback.py b/pywb/apps/wayback.py index 68d042a3..2febc1f4 100644 --- a/pywb/apps/wayback.py +++ b/pywb/apps/wayback.py @@ -1,18 +1,6 @@ -import os +from gevent.monkey import patch_all; patch_all() +from pywb.urlrewrite.frontendapp import FrontEndApp -if os.environ.get('GEVENT_MONKEY_PATCH') == '1': #pragma: no cover - # Use gevent if available - try: - from gevent.monkey import patch_all; patch_all() - print('gevent patched!') - except Exception as e: - pass - -from pywb.framework.wsgi_wrappers import init_app -from pywb.webapp.pywb_init import create_wb_router +application = FrontEndApp() -#================================================================= -# init pywb app -#================================================================= -application = init_app(create_wb_router, load_yaml=True) diff --git a/pywb/apps/webagg.py b/pywb/apps/webagg.py index 49c04aa2..f393533a 100644 --- a/pywb/apps/webagg.py +++ b/pywb/apps/webagg.py @@ -1,6 +1,7 @@ from gevent.monkey import patch_all; patch_all() from pywb.webagg.autoapp import AutoConfigApp -application = AutoConfigApp() +application = AutoConfigApp(custom_config={'collections': {'live': '$live'}}) + diff --git a/pywb/webagg/autoapp.py b/pywb/webagg/autoapp.py index 8da9e604..9d8c7154 100644 --- a/pywb/webagg/autoapp.py +++ b/pywb/webagg/autoapp.py @@ -39,13 +39,16 @@ class AutoConfigApp(ResAggApp): if config_file: try: - custom_config = load_config('PYWB_CONFIG_FILE', config_file) + file_config = load_config('PYWB_CONFIG_FILE', config_file) + config.update(file_config) except Exception as e: if not custom_config: custom_config = {'debug': True} print(e) if custom_config: + if 'collections' in custom_config and 'collections' in config: + custom_config['collections'].update(config['collections']) config.update(custom_config) super(AutoConfigApp, self).__init__(debug=config.get('debug', False)) diff --git a/tests/test_auto_colls.py b/tests/test_auto_colls.py index 6e257432..af816194 100644 --- a/tests/test_auto_colls.py +++ b/tests/test_auto_colls.py @@ -87,17 +87,17 @@ class TestManagedColls(object): """ from pywb.apps.cli import wayback - wayback(['-p', '0', '-s', 'gevent']) + wayback(['-p', '0']) # Nothing to auto-index.. yet with raises(SystemExit): - wayback(['-a', '-p', '0', '-s', 'gevent']) + wayback(['-a', '-p', '0']) colls = os.path.join(self.root_dir, 'collections') os.mkdir(colls) pywb.manager.autoindex.keep_running = False - wayback(['-a', '-p', '0', '-s', 'gevent']) + wayback(['-a', '-p', '0']) def test_create_first_coll(self): """ Test first collection creation, with all required dirs diff --git a/pywb/cdx/test/test_cdxserver.py b/tests_disabled/test_cdxserver.py similarity index 100% rename from pywb/cdx/test/test_cdxserver.py rename to tests_disabled/test_cdxserver.py