backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

add support for optional proxies (verify set to false for now)

Browse Source
This commit is contained in:
Ilya Kreymer 2014-04-13 17:50:26 -07:00
parent 7636c9d3f7
commit d8c9a803f6
1 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pywb/rewrite/rewrite_live.py
View File

@ -50,7 +50,8 @@ class LiveRewriter(object):
def fetch_http(self, url, def fetch_http(self, url,
env=None, env=None,
req_headers={}, req_headers={},
follow_redirects=False): follow_redirects=False,
proxies=None):
method = 'GET' method = 'GET'
data = None data = None
@ -69,7 +70,9 @@ class LiveRewriter(object):
data=data, data=data,
headers=req_headers, headers=req_headers,
allow_redirects=follow_redirects, allow_redirects=follow_redirects,
stream=True) proxies=proxies,
stream=True,
verify=False)
statusline = str(response.status_code) + ' ' + response.reason statusline = str(response.status_code) + ' ' + response.reason
@ -81,8 +84,13 @@ class LiveRewriter(object):
return (status_headers, stream) return (status_headers, stream)
def fetch_request(self, url, urlrewriter, def fetch_request(self, url, urlrewriter,
head_insert_func=None, urlkey=None, head_insert_func=None,
env=None, req_headers={}, follow_redirects=False): urlkey=None,
env=None,
req_headers={},
timestamp=None,
follow_redirects=False,
proxies=None):
ts_err = url.split('///') ts_err = url.split('///')
@ -94,7 +102,8 @@ class LiveRewriter(object):
if is_http(url): if is_http(url):
(status_headers, stream) = self.fetch_http(url, env, req_headers, (status_headers, stream) = self.fetch_http(url, env, req_headers,
follow_redirects) follow_redirects,
proxies)
else: else:
(status_headers, stream) = self.fetch_local_file(url) (status_headers, stream) = self.fetch_local_file(url)
@ -102,8 +111,11 @@ class LiveRewriter(object):
if not urlkey: if not urlkey:
urlkey = canonicalize(url) urlkey = canonicalize(url)
if timestamp is None:
timestamp = datetime_to_timestamp(datetime.datetime.utcnow())
cdx = {'urlkey': urlkey, cdx = {'urlkey': urlkey,
'timestamp': datetime_to_timestamp(datetime.datetime.utcnow()), 'timestamp': timestamp,
'original': url, 'original': url,
'statuscode': status_headers.get_statuscode(), 'statuscode': status_headers.get_statuscode(),
'mimetype': status_headers.get_header('Content-Type') 'mimetype': status_headers.get_header('Content-Type')