backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 08:04:49 +01:00
Code Issues Projects Releases Activity

s3 loader improvements: support AWS cred in username and password part of url, stream s3 response directly

Browse Source
This commit is contained in:
Ilya Kreymer 2016-05-17 18:55:10 -07:00
parent 94afab0bb2
commit 119074e0ee
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pywb/utils/loaders.py
View File

@ -366,24 +366,36 @@ class S3Loader(object):
raise IOError('To load from s3 paths, ' +
'you must install boto: pip install boto')
if not self.s3conn:
try:
self.s3conn = connect_s3()
except Exception: #pragma: no cover
self.s3conn = connect_s3(anon=True)
aws_access_key_id = None
aws_secret_access_key = None
parts = urlsplit(url)
bucket = self.s3conn.get_bucket(parts.netloc)
if parts.username and parts.password:
aws_access_key_id = parts.username
aws_secret_access_key = parts.password
bucket_name = parts.netloc.split('@', 1)[-1]
else:
bucket_name = parts.netloc
headers = {'Range': BlockLoader._make_range_header(offset, length)}
if not self.s3conn:
try:
self.s3conn = connect_s3(aws_access_key_id, aws_secret_access_key)
except Exception: #pragma: no cover
self.s3conn = connect_s3(anon=True)
bucket = self.s3conn.get_bucket(bucket_name)
key = bucket.get_key(parts.path)
result = key.get_contents_as_string(headers=headers)
key.close()
if offset == 0 and length == -1:
headers = {}
else:
headers = {'Range': BlockLoader._make_range_header(offset, length)}
return BytesIO(result)
# Read range
key.open_read(headers=headers)
return key
#=================================================================