backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

live rewrite: proxy setup refactor: ignore_proxy flag, pass proxy during constructor only

Browse Source
This commit is contained in:
Ilya Kreymer 2014-12-22 21:58:07 -08:00
parent b54e4c1c06
commit a8b4041716
2 changed files with 30 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pywb/rewrite/rewrite_live.py
View File

@ -22,11 +22,18 @@ from rewrite_content import RewriteContent
#================================================================= #=================================================================
class LiveRewriter(object): class LiveRewriter(object):
def __init__(self, is_framed_replay=False, default_proxy=None): def __init__(self, is_framed_replay=False, proxies=None):
self.rewriter = RewriteContent(is_framed_replay=is_framed_replay) self.rewriter = RewriteContent(is_framed_replay=is_framed_replay)
self.default_proxy = default_proxy
if self.default_proxy: self.proxies = proxies
logging.debug('Live Rewrite via proxy ' + self.default_proxy)
if self.proxies:
logging.debug('Live Rewrite via proxy ' + str(proxies))
if isinstance(proxies, str):
self.proxies = {'http': proxies,
'https': proxies}
else: else:
logging.debug('Live Rewrite Direct (no proxy)') logging.debug('Live Rewrite Direct (no proxy)')
@ -86,16 +93,14 @@ class LiveRewriter(object):
env=None, env=None,
req_headers=None, req_headers=None,
follow_redirects=False, follow_redirects=False,
proxies=None): ignore_proxies=False):
method = 'GET' method = 'GET'
data = None data = None
if proxies == False: proxies = None
proxies = None if not ignore_proxies:
elif not proxies and self.default_proxy: proxies = self.proxies
proxies = {'http': self.default_proxy,
'https': self.default_proxy}
if not req_headers: if not req_headers:
req_headers = {} req_headers = {}
@ -138,7 +143,7 @@ class LiveRewriter(object):
req_headers={}, req_headers={},
timestamp=None, timestamp=None,
follow_redirects=False, follow_redirects=False,
proxies=None): ignore_proxies=False):
ts_err = url.split('///') ts_err = url.split('///')
@ -153,7 +158,7 @@ class LiveRewriter(object):
if is_http(url): if is_http(url):
(status_headers, stream) = self.fetch_http(url, env, req_headers, (status_headers, stream) = self.fetch_http(url, env, req_headers,
follow_redirects, follow_redirects,
proxies) ignore_proxies)
else: else:
(status_headers, stream) = self.fetch_local_file(url) (status_headers, stream) = self.fetch_local_file(url)

26
pywb/webapp/live_rewrite_handler.py
View File

@ -28,21 +28,21 @@ class RewriteHandler(SearchPageWbUrlHandler):
LIVE_COOKIE = 'pywb.timestamp={0}; max-age=60' LIVE_COOKIE = 'pywb.timestamp={0}; max-age=60'
YT_DL_TYPE = 'application/vnd.youtube-dl_formats+json'
youtubedl = None youtubedl = None
def __init__(self, config): def __init__(self, config):
super(RewriteHandler, self).__init__(config) super(RewriteHandler, self).__init__(config)
self.default_proxy = config.get('proxyhostport') self.proxy = config.get('proxyhostport')
self.rewriter = LiveRewriter(is_framed_replay=self.is_frame_mode, self.rewriter = LiveRewriter(is_framed_replay=self.is_frame_mode,
default_proxy=self.default_proxy) proxies=self.proxy)
self.head_insert_view = HeadInsertView.init_from_config(config) self.head_insert_view = HeadInsertView.init_from_config(config)
self.live_cookie = config.get('live-cookie', self.LIVE_COOKIE) self.live_cookie = config.get('live-cookie', self.LIVE_COOKIE)
self.no_proxy_range = config.get('no_proxy_range', True)
self.ydl = None self.ydl = None
self._cache = None self._cache = None
@ -74,14 +74,14 @@ class RewriteHandler(SearchPageWbUrlHandler):
if ref_wburl_str: if ref_wburl_str:
wbrequest.env['REL_REFERER'] = WbUrl(ref_wburl_str).url wbrequest.env['REL_REFERER'] = WbUrl(ref_wburl_str).url
proxies = None # default ignore_proxies = False
use_206 = False use_206 = False
url = None url = None
readd_range = False readd_range = False
cache_key = None cache_key = None
if self.default_proxy and self.no_proxy_range: if self.proxy:
rangeres = wbrequest.extract_range() rangeres = wbrequest.extract_range()
if rangeres: if rangeres:
@ -94,7 +94,7 @@ class RewriteHandler(SearchPageWbUrlHandler):
readd_range = True readd_range = True
else: else:
# disables proxy # disables proxy
proxies = False ignore_proxies = True
# sets cache_key only if not already cached # sets cache_key only if not already cached
cache_key = self._check_url_cache(url) cache_key = self._check_url_cache(url)
@ -104,7 +104,7 @@ class RewriteHandler(SearchPageWbUrlHandler):
head_insert_func=head_insert_func, head_insert_func=head_insert_func,
req_headers=req_headers, req_headers=req_headers,
env=wbrequest.env, env=wbrequest.env,
proxies=proxies) ignore_proxies=ignore_proxies)
wbresponse = self._make_response(wbrequest, *result) wbresponse = self._make_response(wbrequest, *result)
@ -150,8 +150,8 @@ class RewriteHandler(SearchPageWbUrlHandler):
referrer = wbrequest.env.get('REL_REFERER') referrer = wbrequest.env.get('REL_REFERER')
def do_ping(): def do_ping():
proxies = {'http': self.default_proxy, proxies = {'http': self.proxy,
'https': self.default_proxy} 'https': self.proxy}
headers = self._live_request_headers(wbrequest) headers = self._live_request_headers(wbrequest)
headers['Connection'] = 'close' headers['Connection'] = 'close'
@ -201,11 +201,11 @@ class RewriteHandler(SearchPageWbUrlHandler):
info = self.youtubedl.extract_info(video_url) info = self.youtubedl.extract_info(video_url)
content_type = 'application/vnd.youtube-dl_formats+json' content_type = self.YT_DL_TYPE
metadata = json.dumps(info) metadata = json.dumps(info)
if self.default_proxy: if self.proxy:
proxies = {'http': self.default_proxy} proxies = {'http': self.proxy}
headers = self._live_request_headers(wbrequest) headers = self._live_request_headers(wbrequest)
headers['Content-Type'] = content_type headers['Content-Type'] = content_type