mirror of
https://github.com/webrecorder/pywb.git
synced 2025-03-19 18:29:37 +01:00
e1cad621b9
* dedup improvements on top of #597, work towards patching support (#601) - single key 'dedup_policy' of 'skip', 'revisit', 'keep' - optional 'dedup_index_url', defaults to redis urls - support for 'cache: always' to further add cacheing on all requests that have a referrer - updated docs to mention latest config, explain 'instant replay' that is possible when dedup_policy is set - add check to ensure only redis:// URLs can be set for dedup_index_url for now - config: convert shorthand 'recorder: <source_coll>' setting string to dict, don't override custom config
808 lines
32 KiB
Python
808 lines
32 KiB
Python
from gevent.monkey import patch_all; patch_all()
|
|
|
|
from werkzeug.routing import Map, Rule, RequestRedirect, Submount
|
|
from werkzeug.wsgi import pop_path_info
|
|
from six.moves.urllib.parse import urljoin
|
|
from six import iteritems
|
|
from warcio.utils import to_native_str
|
|
from warcio.timeutils import iso_date_to_timestamp
|
|
from wsgiprox.wsgiprox import WSGIProxMiddleware
|
|
|
|
from pywb.recorder.multifilewarcwriter import MultiFileWARCWriter
|
|
from pywb.recorder.recorderapp import RecorderApp
|
|
from pywb.recorder.filters import SkipDupePolicy, WriteDupePolicy, WriteRevisitDupePolicy
|
|
from pywb.recorder.redisindexer import WritableRedisIndexer
|
|
|
|
from pywb.utils.loaders import load_yaml_config
|
|
from pywb.utils.geventserver import GeventServer
|
|
from pywb.utils.io import StreamIter
|
|
from pywb.utils.wbexception import WbException, AppPageNotFound
|
|
|
|
from pywb.warcserver.warcserver import WarcServer
|
|
|
|
from pywb.rewrite.templateview import BaseInsertView
|
|
|
|
from pywb.apps.static_handler import StaticHandler
|
|
from pywb.apps.rewriterapp import RewriterApp
|
|
from pywb.apps.wbrequestresponse import WbResponse
|
|
|
|
import os
|
|
import re
|
|
|
|
import traceback
|
|
import requests
|
|
import logging
|
|
|
|
|
|
# ============================================================================
|
|
class FrontEndApp(object):
|
|
"""Orchestrates pywb's core Wayback Machine functionality and is comprised of 2 core sub-apps and 3 optional apps.
|
|
|
|
Sub-apps:
|
|
- WarcServer: Serves the archive content (WARC/ARC and index) as well as from the live web in record/proxy mode
|
|
- RewriterApp: Rewrites the content served by pywb (if it is to be rewritten)
|
|
- WSGIProxMiddleware (Optional): If proxy mode is enabled, performs pywb's HTTP(s) proxy functionality
|
|
- AutoIndexer (Optional): If auto-indexing is enabled for the collections it is started here
|
|
- RecorderApp (Optional): Recording functionality, available when recording mode is enabled
|
|
|
|
The RewriterApp is configurable and can be set via the class var `REWRITER_APP_CLS`, defaults to RewriterApp
|
|
"""
|
|
|
|
REPLAY_API = 'http://localhost:%s/{coll}/resource/postreq'
|
|
CDX_API = 'http://localhost:%s/{coll}/index'
|
|
RECORD_SERVER = 'http://localhost:%s'
|
|
RECORD_API = 'http://localhost:%s/%s/resource/postreq?param.recorder.coll={coll}'
|
|
|
|
RECORD_ROUTE = '/record'
|
|
|
|
PROXY_CA_NAME = 'pywb HTTPS Proxy CA'
|
|
|
|
PROXY_CA_PATH = os.path.join('proxy-certs', 'pywb-ca.pem')
|
|
|
|
REWRITER_APP_CLS = RewriterApp
|
|
|
|
ALL_DIGITS = re.compile(r'^\d+$')
|
|
|
|
def __init__(self, config_file=None, custom_config=None):
|
|
"""
|
|
:param str|None config_file: Path to the config file
|
|
:param dict|None custom_config: Dictionary containing additional configuration information
|
|
"""
|
|
config_file = config_file or './config.yaml'
|
|
self.handler = self.handle_request
|
|
self.warcserver = WarcServer(config_file=config_file,
|
|
custom_config=custom_config)
|
|
self.recorder = None
|
|
self.recorder_path = None
|
|
self.proxy_default_timestamp = None
|
|
|
|
config = self.warcserver.config
|
|
|
|
self.debug = config.get('debug', False)
|
|
|
|
self.warcserver_server = GeventServer(self.warcserver, port=0)
|
|
|
|
self.proxy_prefix = None # the URL prefix to be used for the collection with proxy mode (e.g. /coll/id_/)
|
|
self.proxy_coll = None # the name of the collection that has proxy mode enabled
|
|
self.proxy_record = False # indicate if proxy recording
|
|
self.init_proxy(config)
|
|
|
|
self.init_recorder(config.get('recorder'))
|
|
|
|
self.init_autoindex(config.get('autoindex'))
|
|
|
|
static_path = config.get('static_url_path', 'pywb/static/').replace('/', os.path.sep)
|
|
self.static_handler = StaticHandler(static_path)
|
|
|
|
self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx')
|
|
self.query_limit = config.get('query_limit')
|
|
|
|
upstream_paths = self.get_upstream_paths(self.warcserver_server.port)
|
|
|
|
framed_replay = config.get('framed_replay', True)
|
|
self.rewriterapp = self.REWRITER_APP_CLS(framed_replay,
|
|
config=config,
|
|
paths=upstream_paths)
|
|
|
|
self.templates_dir = config.get('templates_dir', 'templates')
|
|
self.static_dir = config.get('static_dir', 'static')
|
|
|
|
metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}', 'metadata.yaml')
|
|
self.metadata_cache = MetadataCache(metadata_templ)
|
|
|
|
self._init_routes()
|
|
|
|
def _init_routes(self):
|
|
"""Initialize the routes and based on the configuration file makes available
|
|
specific routes (proxy mode, record)
|
|
"""
|
|
self.url_map = Map()
|
|
self.url_map.add(Rule('/static/_/<coll>/<path:filepath>', endpoint=self.serve_static))
|
|
self.url_map.add(Rule('/static/<path:filepath>', endpoint=self.serve_static))
|
|
self.url_map.add(Rule('/collinfo.json', endpoint=self.serve_listing))
|
|
|
|
if self.is_valid_coll('$root'):
|
|
coll_prefix = ''
|
|
else:
|
|
coll_prefix = '/<coll>'
|
|
self.url_map.add(Rule('/', endpoint=self.serve_home))
|
|
|
|
self._init_coll_routes(coll_prefix)
|
|
|
|
if self.proxy_prefix is not None:
|
|
# Add the proxy-fetch endpoint to enable PreservationWorker to make CORS fetches worry free in proxy mode
|
|
self.url_map.add(Rule('/proxy-fetch/<path:url>', endpoint=self.proxy_fetch,
|
|
methods=['GET', 'HEAD', 'OPTIONS']))
|
|
|
|
def _init_coll_routes(self, coll_prefix):
|
|
"""Initialize and register the routes for specified collection path
|
|
|
|
:param str coll_prefix: The collection path
|
|
:rtype: None
|
|
"""
|
|
routes = self._make_coll_routes(coll_prefix)
|
|
|
|
# init loc routes, if any
|
|
loc_keys = list(self.rewriterapp.loc_map.keys())
|
|
if loc_keys:
|
|
routes.append(Rule('/', endpoint=self.serve_home))
|
|
|
|
submount_route = ', '.join(loc_keys)
|
|
submount_route = '/<any({0}):lang>'.format(submount_route)
|
|
|
|
self.url_map.add(Submount(submount_route, routes))
|
|
|
|
for route in routes:
|
|
self.url_map.add(route)
|
|
|
|
def _make_coll_routes(self, coll_prefix):
|
|
"""Creates a list of standard collection routes for the
|
|
specified collection path
|
|
|
|
:param str coll_prefix: The collection path
|
|
:return: A list of route rules for the supplied collection
|
|
:rtype: list[Rule]
|
|
"""
|
|
routes = [
|
|
Rule(coll_prefix + self.cdx_api_endpoint, endpoint=self.serve_cdx),
|
|
Rule(coll_prefix + '/', endpoint=self.serve_coll_page),
|
|
Rule(coll_prefix + '/timemap/<timemap_output>/<path:url>', endpoint=self.serve_content),
|
|
Rule(coll_prefix + '/<path:url>', endpoint=self.serve_content)
|
|
]
|
|
|
|
if self.recorder_path:
|
|
routes.append(Rule(coll_prefix + self.RECORD_ROUTE + '/<path:url>', endpoint=self.serve_record))
|
|
|
|
return routes
|
|
|
|
def get_upstream_paths(self, port):
|
|
"""Retrieve a dictionary containing the full URLs of the upstream apps
|
|
|
|
:param int port: The port used by the replay and cdx servers
|
|
:return: A dictionary containing the upstream paths (replay, cdx-server, record [if enabled])
|
|
:rtype: dict[str, str]
|
|
"""
|
|
base_paths = {
|
|
'replay': self.REPLAY_API % port,
|
|
'cdx-server': self.CDX_API % port,
|
|
}
|
|
|
|
if self.recorder_path:
|
|
base_paths['record'] = self.recorder_path
|
|
|
|
return base_paths
|
|
|
|
def init_recorder(self, recorder_config):
|
|
"""Initialize the recording functionality of pywb. If recording_config is None this function is a no op
|
|
|
|
:param str|dict|None recorder_config: The configuration for the recorder app
|
|
:rtype: None
|
|
"""
|
|
if not recorder_config:
|
|
self.recorder = None
|
|
self.recorder_path = None
|
|
return
|
|
|
|
if isinstance(recorder_config, str):
|
|
recorder_coll = recorder_config
|
|
recorder_config = {}
|
|
else:
|
|
recorder_coll = recorder_config['source_coll']
|
|
|
|
# cache mode
|
|
self.rec_cache_mode = recorder_config.get('cache', 'default')
|
|
|
|
dedup_policy = recorder_config.get('dedup_policy')
|
|
dedup_by_url = False
|
|
|
|
if dedup_policy == 'none':
|
|
dedup_policy = ''
|
|
|
|
if dedup_policy == 'keep':
|
|
dedup_policy = WriteDupePolicy()
|
|
elif dedup_policy == 'revisit':
|
|
dedup_policy = WriteRevisitDupePolicy()
|
|
elif dedup_policy == 'skip':
|
|
dedup_policy = SkipDupePolicy()
|
|
dedup_by_url = True
|
|
elif dedup_policy:
|
|
msg = 'Invalid option for dedup_policy: {0}'
|
|
raise Exception(msg.format(dedup_policy))
|
|
|
|
if dedup_policy:
|
|
dedup_index = WritableRedisIndexer(redis_url=self.warcserver.dedup_index_url,
|
|
dupe_policy=dedup_policy,
|
|
rel_path_template=self.warcserver.root_dir + '/{coll}/archive')
|
|
else:
|
|
dedup_index = None
|
|
|
|
|
|
warc_writer = MultiFileWARCWriter(self.warcserver.archive_paths,
|
|
max_size=int(recorder_config.get('rollover_size', 1000000000)),
|
|
max_idle_secs=int(recorder_config.get('rollover_idle_secs', 600)),
|
|
filename_template=recorder_config.get('filename_template'),
|
|
dedup_index=dedup_index,
|
|
dedup_by_url=dedup_by_url)
|
|
|
|
self.recorder = RecorderApp(self.RECORD_SERVER % str(self.warcserver_server.port), warc_writer,
|
|
accept_colls=recorder_config.get('source_filter'))
|
|
|
|
recorder_server = GeventServer(self.recorder, port=0)
|
|
|
|
self.recorder_path = self.RECORD_API % (recorder_server.port, recorder_coll)
|
|
|
|
def init_autoindex(self, auto_interval):
|
|
"""Initialize and start the auto-indexing of the collections. If auto_interval is None this is a no op.
|
|
|
|
:param str|int auto_interval: The auto-indexing interval from the configuration file or CLI argument
|
|
"""
|
|
if not auto_interval:
|
|
return
|
|
|
|
from pywb.manager.autoindex import AutoIndexer
|
|
|
|
colls_dir = self.warcserver.root_dir if self.warcserver.root_dir else None
|
|
|
|
indexer = AutoIndexer(colls_dir=colls_dir, interval=int(auto_interval))
|
|
|
|
if not os.path.isdir(indexer.root_path):
|
|
msg = 'No managed directory "{0}" for auto-indexing'
|
|
logging.error(msg.format(indexer.root_path))
|
|
import sys
|
|
sys.exit(2)
|
|
|
|
msg = 'Auto-Indexing Enabled on "{0}", checking every {1} secs'
|
|
logging.info(msg.format(indexer.root_path, auto_interval))
|
|
indexer.start()
|
|
|
|
def is_proxy_enabled(self, environ):
|
|
"""Returns T/F indicating if proxy mode is enabled
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:return: T/F indicating if proxy mode is enabled
|
|
:rtype: bool
|
|
"""
|
|
return self.proxy_prefix is not None and 'wsgiprox.proxy_host' in environ
|
|
|
|
def serve_home(self, environ):
|
|
"""Serves the home (/) view of pywb (not a collections)
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:return: The WbResponse for serving the home (/) path
|
|
:rtype: WbResponse
|
|
"""
|
|
home_view = BaseInsertView(self.rewriterapp.jinja_env, 'index.html')
|
|
fixed_routes = self.warcserver.list_fixed_routes()
|
|
dynamic_routes = self.warcserver.list_dynamic_routes()
|
|
|
|
routes = fixed_routes + dynamic_routes
|
|
|
|
all_metadata = self.metadata_cache.get_all(dynamic_routes)
|
|
|
|
content = home_view.render_to_string(environ,
|
|
routes=routes,
|
|
all_metadata=all_metadata)
|
|
|
|
return WbResponse.text_response(content, content_type='text/html; charset="utf-8"')
|
|
|
|
def serve_static(self, environ, coll='', filepath=''):
|
|
"""Serve a static file associated with a specific collection or one of pywb's own static assets
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The collection the static file is associated with
|
|
:param str filepath: The file path (relative to the collection) for the static assest
|
|
:return: The WbResponse for the static asset
|
|
:rtype: WbResponse
|
|
"""
|
|
proxy_enabled = self.is_proxy_enabled(environ)
|
|
if proxy_enabled and environ.get('REQUEST_METHOD') == 'OPTIONS':
|
|
return WbResponse.options_response(environ)
|
|
if coll:
|
|
path = os.path.join(self.warcserver.root_dir, coll, self.static_dir)
|
|
else:
|
|
path = self.static_dir
|
|
|
|
environ['pywb.static_dir'] = path
|
|
try:
|
|
response = self.static_handler(environ, filepath)
|
|
if proxy_enabled:
|
|
response.add_access_control_headers(env=environ)
|
|
return response
|
|
except Exception:
|
|
self.raise_not_found(environ, 'static_file_not_found', filepath)
|
|
|
|
def get_coll_config(self, coll):
|
|
"""Retrieve the collection config, including metadata, associated with a collection
|
|
|
|
:param str coll: The name of the collection to receive config info for
|
|
:return: The collections config
|
|
:rtype: dict
|
|
"""
|
|
coll_config = {'coll': coll,
|
|
'type': 'replay'}
|
|
|
|
if coll in self.warcserver.list_fixed_routes():
|
|
coll_config.update(self.warcserver.get_coll_config(coll))
|
|
else:
|
|
coll_config['metadata'] = self.metadata_cache.load(coll) or {}
|
|
|
|
return coll_config
|
|
|
|
def serve_coll_page(self, environ, coll='$root'):
|
|
"""Render and serve a collections search page (search.html).
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The name of the collection to serve the collections search page for
|
|
:return: The WbResponse containing the collections search page
|
|
:rtype: WbResponse
|
|
"""
|
|
if not self.is_valid_coll(coll):
|
|
self.raise_not_found(environ, 'coll_not_found', coll)
|
|
|
|
self.setup_paths(environ, coll)
|
|
|
|
coll_config = self.get_coll_config(coll)
|
|
metadata = coll_config.get('metadata')
|
|
|
|
view = BaseInsertView(self.rewriterapp.jinja_env, 'search.html')
|
|
|
|
wb_prefix = environ.get('SCRIPT_NAME', '')
|
|
if wb_prefix:
|
|
wb_prefix += '/'
|
|
|
|
content = view.render_to_string(environ,
|
|
wb_prefix=wb_prefix,
|
|
coll=coll,
|
|
coll_config=coll_config,
|
|
metadata=metadata)
|
|
|
|
return WbResponse.text_response(content, content_type='text/html; charset="utf-8"')
|
|
|
|
def serve_cdx(self, environ, coll='$root'):
|
|
"""Make the upstream CDX query for a collection and response with the results of the query
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The name of the collection this CDX query is for
|
|
:return: The WbResponse containing the results of the CDX query
|
|
:rtype: WbResponse
|
|
"""
|
|
base_url = self.rewriterapp.paths['cdx-server']
|
|
|
|
# if coll == self.all_coll:
|
|
# coll = '*'
|
|
|
|
cdx_url = base_url.format(coll=coll)
|
|
|
|
if environ.get('QUERY_STRING'):
|
|
cdx_url += '&' if '?' in cdx_url else '?'
|
|
cdx_url += environ.get('QUERY_STRING')
|
|
|
|
if self.query_limit:
|
|
cdx_url += '&' if '?' in cdx_url else '?'
|
|
cdx_url += 'limit=' + str(self.query_limit)
|
|
|
|
try:
|
|
res = requests.get(cdx_url, stream=True)
|
|
|
|
content_type = res.headers.get('Content-Type')
|
|
|
|
return WbResponse.bin_stream(StreamIter(res.raw),
|
|
content_type=content_type)
|
|
|
|
except Exception as e:
|
|
return WbResponse.text_response('Error: ' + str(e), status='400 Bad Request')
|
|
|
|
def serve_record(self, environ, coll='$root', url=''):
|
|
"""Serve a URL's content from a WARC/ARC record in replay mode or from the live web in
|
|
live, proxy, and record mode.
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The name of the collection the record is to be served from
|
|
:param str url: The URL for the corresponding record to be served if it exists
|
|
:return: WbResponse containing the contents of the record/URL
|
|
:rtype: WbResponse
|
|
"""
|
|
if coll in self.warcserver.list_fixed_routes():
|
|
return WbResponse.text_response('Error: Can Not Record Into Custom Collection "{0}"'.format(coll))
|
|
|
|
return self.serve_content(environ, coll, url, record=True)
|
|
|
|
def serve_content(self, environ, coll='$root', url='', timemap_output='', record=False):
|
|
"""Serve the contents of a URL/Record rewriting the contents of the response when applicable.
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The name of the collection the record is to be served from
|
|
:param str url: The URL for the corresponding record to be served if it exists
|
|
:param str timemap_output: The contents of the timemap included in the link header of the response
|
|
:param bool record: Should the content being served by recorded (save to a warc). Only valid in record mode
|
|
:return: WbResponse containing the contents of the record/URL
|
|
:rtype: WbResponse
|
|
"""
|
|
if not self.is_valid_coll(coll):
|
|
self.raise_not_found(environ, 'coll_not_found', coll)
|
|
|
|
self.setup_paths(environ, coll, record)
|
|
|
|
request_uri = environ.get('REQUEST_URI')
|
|
script_name = environ.get('SCRIPT_NAME', '') + '/'
|
|
if request_uri and request_uri.startswith(script_name):
|
|
wb_url_str = request_uri[len(script_name):]
|
|
|
|
else:
|
|
wb_url_str = to_native_str(url)
|
|
|
|
if environ.get('QUERY_STRING'):
|
|
wb_url_str += '?' + environ.get('QUERY_STRING')
|
|
|
|
coll_config = self.get_coll_config(coll)
|
|
if record:
|
|
coll_config['type'] = 'record'
|
|
coll_config['cache'] = self.rec_cache_mode
|
|
|
|
if timemap_output:
|
|
coll_config['output'] = timemap_output
|
|
# ensure that the timemap path information is not included
|
|
wb_url_str = wb_url_str.replace('timemap/{0}/'.format(timemap_output), '')
|
|
|
|
return self.rewriterapp.render_content(wb_url_str, coll_config, environ)
|
|
|
|
def setup_paths(self, environ, coll, record=False):
|
|
"""Populates the WSGI environment dictionary with the path information necessary to perform a response for
|
|
content or record.
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str coll: The name of the collection the record is to be served from
|
|
:param bool record: Should the content being served by recorded (save to a warc). Only valid in record mode
|
|
"""
|
|
if not coll or not self.warcserver.root_dir:
|
|
return
|
|
|
|
if coll != '$root':
|
|
pop_path_info(environ)
|
|
if record:
|
|
pop_path_info(environ)
|
|
|
|
paths = [self.warcserver.root_dir]
|
|
|
|
if coll != '$root':
|
|
paths.append(coll)
|
|
|
|
paths.append(self.templates_dir)
|
|
|
|
# jinja2 template paths always use '/' as separator
|
|
environ['pywb.templates_dir'] = '/'.join(paths)
|
|
|
|
def serve_listing(self, environ):
|
|
"""Serves the response for WARCServer fixed and dynamic listing (paths)
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:return: WbResponse containing the frontend apps WARCServer URL paths
|
|
:rtype: WbResponse
|
|
"""
|
|
result = {'fixed': self.warcserver.list_fixed_routes(),
|
|
'dynamic': self.warcserver.list_dynamic_routes()
|
|
}
|
|
|
|
return WbResponse.json_response(result)
|
|
|
|
def is_valid_coll(self, coll):
|
|
"""Determines if the collection name for a request is valid (exists)
|
|
|
|
:param str coll: The name of the collection to check
|
|
:return: True if the collection is valid, false otherwise
|
|
:rtype: bool
|
|
"""
|
|
# if coll == self.all_coll:
|
|
# return True
|
|
|
|
return (coll in self.warcserver.list_fixed_routes() or
|
|
coll in self.warcserver.list_dynamic_routes())
|
|
|
|
def raise_not_found(self, environ, err_type, url):
|
|
"""Utility function for raising a werkzeug.exceptions.NotFound execption with the supplied WSGI environment
|
|
and message.
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param str err_type: The identifier for type of error that occured
|
|
:param str url: The url of the archived page that was requested
|
|
"""
|
|
raise AppPageNotFound(err_type, url)
|
|
|
|
def _check_refer_redirect(self, environ):
|
|
"""Returns a WbResponse for a HTTP 307 redirection if the HTTP referer header is the same as the HTTP host header
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:return: WbResponse HTTP 307 redirection
|
|
:rtype: WbResponse
|
|
"""
|
|
referer = environ.get('HTTP_REFERER')
|
|
if not referer:
|
|
return
|
|
|
|
host = environ.get('HTTP_HOST')
|
|
if host not in referer:
|
|
return
|
|
|
|
inx = referer[1:].find('http')
|
|
if not inx:
|
|
inx = referer[1:].find('///')
|
|
|
|
if inx < 0:
|
|
return
|
|
|
|
url = referer[inx + 1:]
|
|
host = referer[:inx + 1]
|
|
|
|
orig_url = environ['PATH_INFO']
|
|
if environ.get('QUERY_STRING'):
|
|
orig_url += '?' + environ['QUERY_STRING']
|
|
|
|
full_url = host + urljoin(url, orig_url)
|
|
return WbResponse.redir_response(full_url, '307 Redirect')
|
|
|
|
def __call__(self, environ, start_response):
|
|
"""Handles a request
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param start_response:
|
|
:return: The WbResponse for the request
|
|
:rtype: WbResponse
|
|
"""
|
|
return self.handler(environ, start_response)
|
|
|
|
def handle_request(self, environ, start_response):
|
|
"""Retrieves the route handler and calls the handler returning its the response
|
|
|
|
:param dict environ: The WSGI environment dictionary for the request
|
|
:param start_response:
|
|
:return: The WbResponse for the request
|
|
:rtype: WbResponse
|
|
"""
|
|
urls = self.url_map.bind_to_environ(environ)
|
|
try:
|
|
endpoint, args = urls.match()
|
|
# store original script_name (original prefix) before modifications are made
|
|
environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '')
|
|
|
|
# store original script_name (original prefix) before modifications are made
|
|
environ['ORIG_SCRIPT_NAME'] = environ.get('SCRIPT_NAME')
|
|
|
|
lang = args.pop('lang', '')
|
|
if lang:
|
|
pop_path_info(environ)
|
|
environ['pywb_lang'] = lang
|
|
|
|
response = endpoint(environ, **args)
|
|
|
|
except RequestRedirect as rr:
|
|
# if werkzeug throws this, likely a missing slash redirect
|
|
# also check referrer here to avoid another redirect later
|
|
redir = self._check_refer_redirect(environ)
|
|
if redir:
|
|
return redir(environ, start_response)
|
|
|
|
response = WbResponse.redir_response(rr.new_url, '307 Redirect')
|
|
|
|
except WbException as wbe:
|
|
if wbe.status_code == 404:
|
|
redir = self._check_refer_redirect(environ)
|
|
if redir:
|
|
return redir(environ, start_response)
|
|
|
|
response = self.rewriterapp.handle_error(environ, wbe)
|
|
|
|
except Exception as e:
|
|
if self.debug:
|
|
traceback.print_exc()
|
|
|
|
response = self.rewriterapp._error_response(environ, WbException('Internal Error: ' + str(e)))
|
|
|
|
return response(environ, start_response)
|
|
|
|
@classmethod
|
|
def create_app(cls, port):
|
|
"""Create a new instance of FrontEndApp that listens on port with a hostname of 0.0.0.0
|
|
|
|
:param int port: The port FrontEndApp is to listen on
|
|
:return: A new instance of FrontEndApp wrapped in GeventServer
|
|
:rtype: GeventServer
|
|
"""
|
|
app = FrontEndApp()
|
|
app_server = GeventServer(app, port=port, hostname='0.0.0.0')
|
|
return app_server
|
|
|
|
def init_proxy(self, config):
|
|
"""Initialize and start proxy mode. If proxy configuration entry is not contained in the config
|
|
this is a no op. Causes handler to become an instance of WSGIProxMiddleware.
|
|
|
|
:param dict config: The configuration object used to configure this instance of FrontEndApp
|
|
"""
|
|
proxy_config = config.get('proxy')
|
|
if not proxy_config:
|
|
return
|
|
|
|
if isinstance(proxy_config, str):
|
|
proxy_coll = proxy_config
|
|
proxy_config = {}
|
|
else:
|
|
proxy_coll = proxy_config['coll']
|
|
|
|
if '/' in proxy_coll:
|
|
raise Exception('Proxy collection can not contain "/"')
|
|
|
|
proxy_config['ca_name'] = proxy_config.get('ca_name', self.PROXY_CA_NAME)
|
|
proxy_config['ca_file_cache'] = proxy_config.get('ca_file_cache', self.PROXY_CA_PATH)
|
|
|
|
if proxy_config.get('recording'):
|
|
logging.info('Proxy recording into collection "{0}"'.format(proxy_coll))
|
|
if proxy_coll in self.warcserver.list_fixed_routes():
|
|
raise Exception('Can not record into fixed collection')
|
|
|
|
proxy_route = proxy_coll + self.RECORD_ROUTE
|
|
if not config.get('recorder'):
|
|
config['recorder'] = 'live'
|
|
|
|
self.proxy_record = True
|
|
|
|
else:
|
|
logging.info('Proxy enabled for collection "{0}"'.format(proxy_coll))
|
|
self.proxy_record = False
|
|
proxy_route = proxy_coll
|
|
|
|
if proxy_config.get('enable_content_rewrite', True):
|
|
self.proxy_prefix = '/{0}/bn_/'.format(proxy_route)
|
|
else:
|
|
self.proxy_prefix = '/{0}/id_/'.format(proxy_route)
|
|
|
|
self.proxy_default_timestamp = proxy_config.get('default_timestamp')
|
|
if self.proxy_default_timestamp:
|
|
if not self.ALL_DIGITS.match(self.proxy_default_timestamp):
|
|
try:
|
|
self.proxy_default_timestamp = iso_date_to_timestamp(self.proxy_default_timestamp)
|
|
except Exception:
|
|
raise Exception('Invalid Proxy Timestamp: Must Be All-Digit Timestamp or ISO Date Format')
|
|
|
|
self.proxy_coll = proxy_coll
|
|
|
|
self.handler = WSGIProxMiddleware(self.handle_request,
|
|
self.proxy_route_request,
|
|
proxy_host=proxy_config.get('host', 'pywb.proxy'),
|
|
proxy_options=proxy_config)
|
|
|
|
def proxy_route_request(self, url, environ):
|
|
""" Return the full url that this proxy request will be routed to
|
|
The 'environ' PATH_INFO and REQUEST_URI will be modified based on the returned url
|
|
|
|
Default is to use the 'proxy_prefix' to point to the proxy collection
|
|
"""
|
|
if self.proxy_default_timestamp:
|
|
environ['pywb_proxy_default_timestamp'] = self.proxy_default_timestamp
|
|
|
|
return self.proxy_prefix + url
|
|
|
|
def proxy_fetch(self, env, url):
|
|
"""Proxy mode only endpoint that handles OPTIONS requests and COR fetches for Preservation Worker.
|
|
|
|
Due to normal cross-origin browser restrictions in proxy mode, auto fetch worker cannot access the CSS rules
|
|
of cross-origin style sheets and must re-fetch them in a manner that is CORS safe. This endpoint facilitates
|
|
that by fetching the stylesheets for the auto fetch worker and then responds with its contents
|
|
|
|
:param dict env: The WSGI environment dictionary
|
|
:param str url: The URL of the resource to be fetched
|
|
:return: WbResponse that is either response to an Options request or the results of fetching url
|
|
:rtype: WbResponse
|
|
"""
|
|
if not self.is_proxy_enabled(env):
|
|
# we are not in proxy mode so just respond with forbidden
|
|
return WbResponse.text_response('proxy mode must be enabled to use this endpoint',
|
|
status='403 Forbidden')
|
|
|
|
if env.get('REQUEST_METHOD') == 'OPTIONS':
|
|
return WbResponse.options_response(env)
|
|
|
|
# ensure full URL
|
|
url = env['REQUEST_URI'].split('/proxy-fetch/', 1)[-1]
|
|
|
|
env['REQUEST_URI'] = self.proxy_prefix + url
|
|
env['PATH_INFO'] = self.proxy_prefix + env['PATH_INFO'].split('/proxy-fetch/', 1)[-1]
|
|
|
|
# make request using normal serve_content
|
|
response = self.serve_content(env, self.proxy_coll, url, record=self.proxy_record)
|
|
|
|
# for WR
|
|
if isinstance(response, WbResponse):
|
|
response.add_access_control_headers(env=env)
|
|
return response
|
|
|
|
|
|
# ============================================================================
|
|
class MetadataCache(object):
|
|
"""This class holds the collection medata template string and
|
|
caches the metadata for a collection once it is rendered once.
|
|
Cached metadata is updated if its corresponding file has been updated since last cache time (file mtime based)"""
|
|
|
|
def __init__(self, template_str):
|
|
"""
|
|
:param str template_str: The template string to be cached
|
|
"""
|
|
self.template_str = template_str
|
|
self.cache = {}
|
|
|
|
def load(self, coll):
|
|
"""Load and receive the metadata associated with a collection.
|
|
|
|
If the metadata for the collection is not cached yet its metadata file is read in and stored.
|
|
If the cache has seen the collection before the mtime of the metadata file is checked and if it is more recent
|
|
than the cached time, the cache is updated and returned otherwise the cached version is returned.
|
|
|
|
:param str coll: Name of a collection
|
|
:return: The cached metadata for a collection
|
|
:rtype: dict
|
|
"""
|
|
path = self.template_str.format(coll=coll)
|
|
try:
|
|
mtime = os.path.getmtime(path)
|
|
obj = self.cache.get(path)
|
|
except Exception:
|
|
return {}
|
|
|
|
if not obj:
|
|
return self.store_new(coll, path, mtime)
|
|
|
|
cached_mtime, data = obj
|
|
if mtime == cached_mtime == mtime:
|
|
return obj
|
|
|
|
return self.store_new(coll, path, mtime)
|
|
|
|
def store_new(self, coll, path, mtime):
|
|
"""Load a collections metadata file and store it
|
|
|
|
:param str coll: The name of the collection the metadata is for
|
|
:param str path: The path to the collections metadata file
|
|
:param float mtime: The current mtime of the collections metadata file
|
|
:return: The collections metadata
|
|
:rtype: dict
|
|
"""
|
|
obj = load_yaml_config(path)
|
|
self.cache[coll] = (mtime, obj)
|
|
return obj
|
|
|
|
def get_all(self, routes):
|
|
"""Load the metadata for all routes (collections) and populate the cache
|
|
|
|
:param list[str] routes: List of collection names
|
|
:return: A dictionary containing each collections metadata
|
|
:rtype: dict
|
|
"""
|
|
for route in routes:
|
|
self.load(route)
|
|
|
|
return {name: value[1] for name, value in iteritems(self.cache)}
|
|
|
|
|
|
# ============================================================================
|
|
if __name__ == "__main__":
|
|
app_server = FrontEndApp.create_app(port=8080)
|
|
app_server.join()
|