diff --git a/pywb/bootstrap/__init__.py b/pywb/bootstrap/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pywb/config_utils.py b/pywb/bootstrap/config_utils.py similarity index 81% rename from pywb/config_utils.py rename to pywb/bootstrap/config_utils.py index 672e8735..2307022a 100644 --- a/pywb/config_utils.py +++ b/pywb/bootstrap/config_utils.py @@ -1,16 +1,16 @@ -import views -import handlers -import replay_views import logging from pywb.warc.recordloader import ArcWarcRecordLoader from pywb.warc.resolvingloader import ResolvingLoader from pywb.rewrite.rewrite_content import RewriteContent +from pywb.core.views import J2TemplateView, J2HtmlCapturesView +from pywb.core.handlers import WBHandler +from pywb.core.replay_views import ReplayView #================================================================= # Config Loading #================================================================= -def load_template_file(file, desc = None, view_class = views.J2TemplateView): +def load_template_file(file, desc = None, view_class = J2TemplateView): if file: logging.debug('Adding {0}: {1}'.format(desc if desc else name, file)) file = view_class(file) @@ -25,7 +25,7 @@ def create_wb_handler(cdx_server, config): resolving_loader = ResolvingLoader(paths = paths, cdx_server = cdx_server, record_loader = record_loader) - replayer = replay_views.ReplayView( + replayer = ReplayView( content_loader = resolving_loader, content_rewriter = RewriteContent(), @@ -40,12 +40,12 @@ def create_wb_handler(cdx_server, config): ) - wb_handler = handlers.WBHandler( + wb_handler = WBHandler( cdx_server, replayer, - html_view = load_template_file(config.get('query_html'), 'Captures Page', views.J2HtmlCapturesView), + html_view = load_template_file(config.get('query_html'), 'Captures Page', J2HtmlCapturesView), search_view = load_template_file(config.get('search_html'), 'Search Page'), ) diff --git a/pywb/pywb_init.py b/pywb/bootstrap/pywb_init.py similarity index 82% rename from pywb/pywb_init.py rename to pywb/bootstrap/pywb_init.py index be4bdded..7465ba64 100644 --- a/pywb/pywb_init.py +++ b/pywb/bootstrap/pywb_init.py @@ -1,8 +1,10 @@ -import handlers -import archivalrouter +from pywb.core.handlers import CDXHandler, StaticHandler +from pywb.core.handlers import DebugEchoHandler, DebugEchoEnvHandler +from pywb.dispatch.archivalrouter import ArchivalRouter, Route +from pywb.dispatch.proxy import ProxyArchivalRouter +from pywb.core.indexreader import IndexReader + import config_utils -import proxy -from indexreader import IndexReader import os import yaml @@ -67,32 +69,32 @@ def pywb_config_manual(passed_config = {}): logging.debug('Adding Collection: ' + name) - route_class = route_config.get('route_class', archivalrouter.Route) + route_class = route_config.get('route_class', Route) routes.append(route_class(name, wb_handler, config = route_config)) # cdx query handler if route_config.get('enable_cdx_api', False): - routes.append(archivalrouter.Route(name + '-cdx', handlers.CDXHandler(cdx_server))) + routes.append(Route(name + '-cdx', CDXHandler(cdx_server))) if config.get('debug_echo_env', False): - routes.append(archivalrouter.Route('echo_env', handlers.DebugEchoEnvHandler())) + routes.append(Route('echo_env', DebugEchoEnvHandler())) if config.get('debug_echo_req', False): - routes.append(archivalrouter.Route('echo_req', handlers.DebugEchoHandler())) + routes.append(Route('echo_req', DebugEchoHandler())) static_routes = config.get('static_routes') for static_name, static_path in static_routes.iteritems(): - routes.append(archivalrouter.Route(static_name, handlers.StaticHandler(static_path))) + routes.append(Route(static_name, StaticHandler(static_path))) # Check for new proxy mode! if config.get('enable_http_proxy', False): - router = proxy.ProxyArchivalRouter + router = ProxyArchivalRouter else: - router = archivalrouter.ArchivalRouter + router = ArchivalRouter # Finally, create wb router return router( diff --git a/pywb/wbapp.py b/pywb/bootstrap/wbapp.py similarity index 92% rename from pywb/wbapp.py rename to pywb/bootstrap/wbapp.py index 0befa172..f9a6d359 100644 --- a/pywb/wbapp.py +++ b/pywb/bootstrap/wbapp.py @@ -1,5 +1,5 @@ -from wbexceptions import WbException, NotFoundException, InternalRedirect -from wbrequestresponse import WbResponse, StatusAndHeaders +from pywb.core.wbexceptions import WbException, NotFoundException, InternalRedirect +from pywb.core.wbrequestresponse import WbResponse, StatusAndHeaders from pywb.cdx.cdxserver import CDXException from pywb.warc.recordloader import ArchiveLoadFailed @@ -91,6 +91,10 @@ def handle_exception(env, error_view, exc, print_trace): #================================================================= DEFAULT_CONFIG_FILE = 'config.yaml' +DEFAULT_INIT_MODULE = 'pywb.bootstrap.pywb_init' + + +#================================================================= def main(): try: logging.basicConfig(format = '%(asctime)s: [%(levelname)s]: %(message)s', level = logging.DEBUG) @@ -100,7 +104,7 @@ def main(): if not config_name: # use default module - config_name = 'pywb.pywb_init' + config_name = DEFAULT_INIT_MODULE logging.info('Loading from default config module "{0}"'.format(config_name)) logging.info('') diff --git a/pywb/core/__init__.py b/pywb/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pywb/handlers.py b/pywb/core/handlers.py similarity index 92% rename from pywb/handlers.py rename to pywb/core/handlers.py index c82db7fe..d166e640 100644 --- a/pywb/handlers.py +++ b/pywb/core/handlers.py @@ -33,11 +33,15 @@ class WBHandler(WbUrlHandler): html_view=None, search_view=None): self.index_reader = index_reader + self.replay = replay - self.text_view = TextCapturesView() + self.text_query_view = TextCapturesView() + + self.query_view = html_view + if not self.query_view: + self.query_view = text_query_view - self.html_view = html_view self.search_view = search_view def __call__(self, wbrequest): @@ -49,11 +53,10 @@ class WBHandler(WbUrlHandler): # new special modifier to always show cdx index if wbrequest.wb_url.mod == 'cdx_': - return self.text_view.render_response(wbrequest, cdx_lines) + return self.text_query_view.render_response(wbrequest, cdx_lines) if (wbrequest.wb_url.type == wbrequest.wb_url.QUERY) or (wbrequest.wb_url.type == wbrequest.wb_url.URL_QUERY): - query_view = self.html_view if self.html_view else self.text_view - return query_view.render_response(wbrequest, cdx_lines) + return self.query_view.render_response(wbrequest, cdx_lines) with PerfTimer(wbrequest.env.get('X_PERF'), 'replay') as t: return self.replay(wbrequest, cdx_lines) @@ -92,7 +95,7 @@ class CDXHandler(BaseHandler): # Static Content Handler #================================================================= class StaticHandler(BaseHandler): - def __init__(self, static_path, pkg = __package__): + def __init__(self, static_path, pkg = 'pywb'): mimetypes.init() self.static_path = static_path diff --git a/pywb/indexreader.py b/pywb/core/indexreader.py similarity index 100% rename from pywb/indexreader.py rename to pywb/core/indexreader.py diff --git a/pywb/replay_views.py b/pywb/core/replay_views.py similarity index 85% rename from pywb/replay_views.py rename to pywb/core/replay_views.py index 4c6907eb..dd11ed4c 100644 --- a/pywb/replay_views.py +++ b/pywb/core/replay_views.py @@ -49,7 +49,7 @@ class ReplayView: self._redirect_if_needed(wbrequest, cdx) # one more check for referrer-based self-redirect - self._reject_referrer_self_redirect(wbrequest, status_headers) + self._reject_referrer_self_redirect(wbrequest) response = None @@ -150,25 +150,30 @@ class ReplayView: def _reject_self_redirect(self, wbrequest, cdx, status_headers): - # self-redirect via location + """ + Check if response is a 3xx redirect to the same url + If so, reject this capture to avoid causing redirect loop + """ if status_headers.statusline.startswith('3'): request_url = wbrequest.wb_url.url.lower() location_url = status_headers.get_header('Location').lower() - #TODO: canonicalize before testing? if (UrlRewriter.strip_protocol(request_url) == UrlRewriter.strip_protocol(location_url)): raise CaptureException('Self Redirect: ' + str(cdx)) - def _reject_referrer_self_redirect(self, wbrequest, status_headers): - # at correct timestamp now, but must check for referrer redirect - # indirect self-redirect, via meta-refresh, if referrer is same as current url - if status_headers.statusline.startswith('2'): - # build full url even if using relative-rewriting - request_url = wbrequest.host_prefix + wbrequest.rel_prefix + str(wbrequest.wb_url) - referrer_url = wbrequest.referrer - if (referrer_url and UrlRewriter.strip_protocol(request_url) == UrlRewriter.strip_protocol(referrer_url)): - raise CaptureException('Self Redirect via Referrer: ' + str(wbrequest.wb_url)) - - + def _reject_referrer_self_redirect(self, wbrequest): + """ + Perform final check for referrer based self-redirect. + This method should be called after verifying request timestamp matches capture. + if referrer is same as current url, reject this response and try another capture + """ + if not wbrequest.referrer: + return + # build full url even if using relative-rewriting + request_url = (wbrequest.host_prefix + + wbrequest.rel_prefix + str(wbrequest.wb_url)) + if (UrlRewriter.strip_protocol(request_url) == + UrlRewriter.strip_protocol(wbrequest.referrer)): + raise CaptureException('Self Redirect via Referrer: ' + str(wbrequest.wb_url)) diff --git a/pywb/test/test_wbrequestresponse.py b/pywb/core/test/test_wbrequestresponse.py similarity index 98% rename from pywb/test/test_wbrequestresponse.py rename to pywb/core/test/test_wbrequestresponse.py index 600ec926..09017564 100644 --- a/pywb/test/test_wbrequestresponse.py +++ b/pywb/core/test/test_wbrequestresponse.py @@ -41,7 +41,7 @@ from pywb.rewrite.wburl import WbUrl from pywb.rewrite.url_rewriter import UrlRewriter from pywb.utils.statusandheaders import StatusAndHeaders -from pywb.wbrequestresponse import WbRequest, WbResponse +from pywb.core.wbrequestresponse import WbRequest, WbResponse def print_req_from_uri(request_uri, env={}, use_abs_prefix=False): diff --git a/pywb/views.py b/pywb/core/views.py similarity index 85% rename from pywb/views.py rename to pywb/core/views.py index f693d1e6..961d1af7 100644 --- a/pywb/views.py +++ b/pywb/core/views.py @@ -1,6 +1,6 @@ -import pywb.utils.timeutils as timeutils +from pywb.utils.timeutils import timestamp_to_datetime +from wbrequestresponse import WbResponse -import wbrequestresponse import urlparse import time @@ -18,7 +18,7 @@ class StaticTextView: return self.text def render_response(self, **kwargs): - return wbrequestresponse.WbResponse.text_stream(self.text) + return WbResponse.text_stream(self.text) #================================================================= class J2TemplateView: @@ -34,7 +34,7 @@ class J2TemplateView: if template_dir.startswith('.') or template_dir.startswith('file://'): loader = FileSystemLoader(template_dir) else: - loader = PackageLoader(__package__, template_dir) + loader = PackageLoader('pywb', template_dir) jinja_env = Environment(loader = loader, trim_blocks = True) jinja_env.filters['format_ts'] = J2TemplateView.format_ts @@ -51,13 +51,13 @@ class J2TemplateView: def render_response(self, **kwargs): template_result = self.render_to_string(**kwargs) status = kwargs.get('status', '200 OK') - return wbrequestresponse.WbResponse.text_response(str(template_result), status = status, content_type = 'text/html; charset=utf-8') + return WbResponse.text_response(str(template_result), status = status, content_type = 'text/html; charset=utf-8') # Filters @staticmethod def format_ts(value, format='%a, %b %d %Y %H:%M:%S'): - value = timeutils.timestamp_to_datetime(value) + value = timestamp_to_datetime(value) return time.strftime(format, value) @staticmethod @@ -90,7 +90,7 @@ class TextCapturesView: cdx += '\n' return cdx cdx_lines = imap(to_str, cdx_lines) - return wbrequestresponse.WbResponse.text_stream(cdx_lines) + return WbResponse.text_stream(cdx_lines) diff --git a/pywb/wbexceptions.py b/pywb/core/wbexceptions.py similarity index 100% rename from pywb/wbexceptions.py rename to pywb/core/wbexceptions.py diff --git a/pywb/wbrequestresponse.py b/pywb/core/wbrequestresponse.py similarity index 100% rename from pywb/wbrequestresponse.py rename to pywb/core/wbrequestresponse.py diff --git a/pywb/dispatch/__init__.py b/pywb/dispatch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pywb/archivalrouter.py b/pywb/dispatch/archivalrouter.py similarity index 98% rename from pywb/archivalrouter.py rename to pywb/dispatch/archivalrouter.py index 4d28b57e..f548969b 100644 --- a/pywb/archivalrouter.py +++ b/pywb/dispatch/archivalrouter.py @@ -1,7 +1,7 @@ import urlparse import re -from wbrequestresponse import WbRequest, WbResponse +from pywb.core.wbrequestresponse import WbRequest, WbResponse from pywb.rewrite.url_rewriter import UrlRewriter diff --git a/pywb/proxy.py b/pywb/dispatch/proxy.py similarity index 98% rename from pywb/proxy.py rename to pywb/dispatch/proxy.py index fc14d1e5..ffc74c47 100644 --- a/pywb/proxy.py +++ b/pywb/dispatch/proxy.py @@ -1,4 +1,4 @@ -from wbrequestresponse import WbResponse, WbRequest +from pywb.core.wbrequestresponse import WbResponse, WbRequest from archivalrouter import ArchivalRouter import urlparse diff --git a/pywb/test/test_archivalrouter.py b/pywb/dispatch/test/test_archivalrouter.py similarity index 97% rename from pywb/test/test_archivalrouter.py rename to pywb/dispatch/test/test_archivalrouter.py index 4379fbfd..82b0d147 100644 --- a/pywb/test/test_archivalrouter.py +++ b/pywb/dispatch/test/test_archivalrouter.py @@ -70,8 +70,8 @@ False """ -from pywb.archivalrouter import Route, ReferRedirect -from pywb.handlers import BaseHandler, WbUrlHandler +from pywb.dispatch.archivalrouter import Route, ReferRedirect +from pywb.core.handlers import BaseHandler, WbUrlHandler import pprint def print_req(req): diff --git a/run.sh b/run.sh index d6e484b9..6232c030 100755 --- a/run.sh +++ b/run.sh @@ -10,14 +10,14 @@ mypath=$(cd `dirname $0` && pwd) # ex: my_pywb.pywb_config() #export 'PYWB_CONFIG=my_pywb' -app="pywb.wbapp" +app="pywb.bootstrap.wbapp" params="--http-socket :8080 -b 65536" #params="--static-map /static=$mypath/static --http-socket :8080 -b 65536" if [ -z "$1" ]; then # Standard root config - params="$params --wsgi pywb.wbapp" + params="$params --wsgi $app" else # run with --mount # requires a file not a package, so creating a mount_run.py to load the package diff --git a/setup.py b/setup.py index 20ac8518..982e067d 100755 --- a/setup.py +++ b/setup.py @@ -11,8 +11,8 @@ setuptools.setup(name='pywb', author_email='ilya@archive.org', long_description=open('README.md').read(), license='GPL', - packages=['pywb','pywb.utils','pywb.cdx','pywb.warc','pywb.rewrite'], - provides=['pywb','pywb.utils','pywb.cdx','pywb.warc','pywb.rewrite'], + packages=['pywb','pywb.utils','pywb.cdx','pywb.warc','pywb.rewrite','pywb.core','pywb.dispatch','pywb.bootstrap'], + provides=['pywb','pywb.utils','pywb.cdx','pywb.warc','pywb.rewrite','pywb.core','pywb.dispatch','pywb.bootstrap'], package_data={'pywb': ['ui/*', 'static/*'], 'pywb.cdx': ['*.yaml']}, data_files = [('sample_archive/cdx/', glob.glob('sample_archive/cdx/*')), ('sample_archive/warcs/', glob.glob('sample_archive/warcs/*'))], diff --git a/tests/test_integration.py b/tests/test_integration.py index 1a7a943c..bede0e2b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,6 @@ import webtest -from pywb.pywb_init import pywb_config -from pywb.wbapp import create_wb_app +from pywb.bootstrap.pywb_init import pywb_config +from pywb.bootstrap.wbapp import create_wb_app from pywb.cdx.cdxobject import CDXObject class TestWb: