backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

apps & cli: remove old apps, keep:

Browse Source 
- webagg-server
- wayback
- live-rewrite-server
support adding custom settings to AutoApp
support for --live flag that automatically adds live-web source at '/live'
tests: disable cdx_server tests as old cdx_server removed
This commit is contained in:
Ilya Kreymer 2017-03-12 12:21:54 -07:00
parent ac84dcc2e3
commit 8fe2c1b5bd
8 changed files with 46 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pywb/apps/cdx_server.py
View File

@ -1,11 +0,0 @@
from pywb.framework.wsgi_wrappers import init_app
#from pywb.core.cdx_api_handler import create_cdx_server_app
from pywb.webapp.pywb_init import create_cdx_server_app
#=================================================================
# init cdx server app
#=================================================================
application = init_app(create_cdx_server_app,
load_yaml=True)

139
pywb/apps/cli.py
View File

@ -1,36 +1,28 @@
from gevent.monkey import patch_all; patch_all()
from argparse import ArgumentParser
import logging
#=================================================================
def cdx_server(args=None): #pragma: no cover
CdxCli(args=args,
default_port=8080,
desc='pywb CDX Index Server').run()
#=============================================================================
def webagg(args=None):
WebaggCli(args=args,
default_port=8070,
desc='pywb Web Aggregator Server').run()
#=================================================================
def live_rewrite_server(args=None): #pragma: no cover
NewLiveCli(args=args,
default_port=8090,
desc='pywb Live Rewrite Proxy Server').run()
#=================================================================
#=============================================================================
def wayback(args=None):
WaybackCli(args=args,
default_port=8080,
desc='pywb Wayback Web Archive Replay').run()
desc='pywb Wayback Machine Server').run()
#=============================================================================
def webagg():
WebaggCli().run()
#=============================================================================
def new_wayback():
NewWaybackCli().run()
def live_rewrite_server(args=None):
LiveCli(args=args,
default_port=8090,
desc='pywb Live Rewrite Proxy Server').run()
#=============================================================================
@ -39,11 +31,13 @@ class BaseCli(object):
parser = ArgumentParser(description=desc)
parser.add_argument('-p', '--port', type=int, default=default_port)
parser.add_argument('-t', '--threads', type=int, default=4)
parser.add_argument('-s', '--server', default='gevent')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--profile', action='store_true')
parser.add_argument('--live', action='store_true', help='Add /live handler')
self.desc = desc
self.extra_config = None
self._extend_parser(parser)
@ -52,14 +46,6 @@ class BaseCli(object):
logging.basicConfig(format='%(asctime)s: [%(levelname)s]: %(message)s',
level=logging.DEBUG if self.r.debug else logging.INFO)
if self.r.server == 'gevent':
try:
from gevent.monkey import patch_all; patch_all()
logging.debug('Using Gevent')
except:
logging.debug('No Gevent')
self.r.server = 'wsgiref'
self.application = self.load()
if self.r.profile:
@ -69,25 +55,12 @@ class BaseCli(object):
def _extend_parser(self, parser): #pragma: no cover
pass
def load(self): #pragma: no cover
pass
def load(self):
if self.r.live:
self.extra_config = {'collections': {'live': '$live'}}
def run(self):
if self.r.server == 'gevent': #pragma: no cover
self.run_gevent()
elif self.r.server == 'waitress': #pragma: no cover
self.run_waitress()
else:
self.run_wsgiref()
def run_waitress(self): #pragma: no cover
from waitress import serve
logging.debug(str(self.desc))
serve(self.application, port=self.r.port, threads=self.r.threads)
def run_wsgiref(self): #pragma: no cover
from pywb.framework.wsgi_wrappers import start_wsgi_ref_server
start_wsgi_ref_server(self.application, self.desc, port=self.r.port)
def run_gevent(self):
from gevent.pywsgi import WSGIServer
@ -95,27 +68,6 @@ class BaseCli(object):
WSGIServer(('', self.r.port), self.application).serve_forever()
#=============================================================================
class LiveCli(BaseCli):
def _extend_parser(self, parser):
parser.add_argument('-x', '--proxy',
help='Specify host:port to use as HTTP/S proxy')
parser.add_argument('-f', '--framed', action='store_true',
help='Replay using framed wrapping mode')
def load(self):
config = dict(proxyhostport=self.r.proxy,
framed_replay='inverse' if self.r.framed else False,
enable_auto_colls=False,
collections={'live': '$liveweb'})
from pywb.webapp.pywb_init import create_wb_router
from pywb.framework.wsgi_wrappers import init_app
return init_app(create_wb_router, load_yaml=False, config=config)
#=============================================================================
class ReplayCli(BaseCli):
def _extend_parser(self, parser):
@ -127,6 +79,7 @@ class ReplayCli(BaseCli):
def load(self):
super(ReplayCli, self).load()
if self.r.directory: #pragma: no cover
os.chdir(self.r.directory)
@ -150,58 +103,32 @@ class ReplayCli(BaseCli):
#=============================================================================
class CdxCli(ReplayCli): #pragma: no cover
class WebaggCli(BaseCli):
def load(self):
from pywb.webapp.pywb_init import create_cdx_server_app
from pywb.framework.wsgi_wrappers import init_app
super(CdxCli, self).load()
return init_app(create_cdx_server_app,
load_yaml=True)
from pywb.webagg.autoapp import AutoConfigApp
super(WebaggCli, self).load()
return AutoConfigApp(custom_config=self.extra_config)
#=============================================================================
class WaybackCli(ReplayCli):
def load(self):
from pywb.webapp.pywb_init import create_wb_router
from pywb.framework.wsgi_wrappers import init_app
from pywb.urlrewrite.frontendapp import FrontEndApp
super(WaybackCli, self).load()
return init_app(create_wb_router,
load_yaml=True)
return FrontEndApp(custom_config=self.extra_config)
#=============================================================================
class WebaggCli(BaseCli):
class LiveCli(BaseCli):
def load(self):
from pywb.apps.webagg import application
return application
def run(self):
self.run_gevent()
#=============================================================================
class NewWaybackCli(ReplayCli):
def load(self):
from pywb.apps.newwayback import application
return application
def run(self):
self.r.server = 'gevent'
super(NewWaybackCli, self).run()
#self.run_gevent()
#=============================================================================
class NewLiveCli(BaseCli):
def load(self):
from pywb.apps.live import application
return application
def run(self):
self.r.server = 'gevent'
super(NewLiveCli, self).run()
#self.run_gevent()
from pywb.urlrewrite.frontendapp import FrontEndApp
self.r.live = True
super(LiveCli, self).load()
return FrontEndApp(config_file=None, custom_config=self.extra_config)
#=============================================================================

6
pywb/apps/newwayback.py
View File

@ -1,6 +0,0 @@
from gevent.monkey import patch_all; patch_all()
from pywb.urlrewrite.frontendapp import FrontEndApp
application = FrontEndApp()

18
pywb/apps/wayback.py
View File

@ -1,18 +1,6 @@
import os
from gevent.monkey import patch_all; patch_all()
from pywb.urlrewrite.frontendapp import FrontEndApp
if os.environ.get('GEVENT_MONKEY_PATCH') == '1': #pragma: no cover
# Use gevent if available
try:
from gevent.monkey import patch_all; patch_all()
print('gevent patched!')
except Exception as e:
pass
from pywb.framework.wsgi_wrappers import init_app
from pywb.webapp.pywb_init import create_wb_router
application = FrontEndApp()
#=================================================================
# init pywb app
#=================================================================
application = init_app(create_wb_router, load_yaml=True)

3
pywb/apps/webagg.py
View File

@ -1,6 +1,7 @@
from gevent.monkey import patch_all; patch_all()
from pywb.webagg.autoapp import AutoConfigApp
application = AutoConfigApp()
application = AutoConfigApp(custom_config={'collections': {'live': '$live'}})

5
pywb/webagg/autoapp.py
View File

@ -39,13 +39,16 @@ class AutoConfigApp(ResAggApp):
if config_file:
try:
custom_config = load_config('PYWB_CONFIG_FILE', config_file)
file_config = load_config('PYWB_CONFIG_FILE', config_file)
config.update(file_config)
except Exception as e:
if not custom_config:
custom_config = {'debug': True}
print(e)
if custom_config:
if 'collections' in custom_config and 'collections' in config:
custom_config['collections'].update(config['collections'])
config.update(custom_config)
super(AutoConfigApp, self).__init__(debug=config.get('debug', False))

6
tests/test_auto_colls.py
View File

@ -87,17 +87,17 @@ class TestManagedColls(object):
"""
from pywb.apps.cli import wayback
wayback(['-p', '0', '-s', 'gevent'])
wayback(['-p', '0'])
# Nothing to auto-index.. yet
with raises(SystemExit):
wayback(['-a', '-p', '0', '-s', 'gevent'])
wayback(['-a', '-p', '0'])
colls = os.path.join(self.root_dir, 'collections')
os.mkdir(colls)
pywb.manager.autoindex.keep_running = False
wayback(['-a', '-p', '0', '-s', 'gevent'])
wayback(['-a', '-p', '0'])
def test_create_first_coll(self):
""" Test first collection creation, with all required dirs

0
pywb/cdx/test/test_cdxserver.py → tests_disabled/test_cdxserver.py
View File