backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

s3 loader improvements: support AWS cred in username and password part of url, stream s3 response directly

Browse Source
This commit is contained in:
Ilya Kreymer 2016-05-17 18:55:10 -07:00
parent 94afab0bb2
commit 119074e0ee
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pywb/utils/loaders.py
View File

@ -366,24 +366,36 @@ class S3Loader(object):
raise IOError('To load from s3 paths, ' + raise IOError('To load from s3 paths, ' +
'you must install boto: pip install boto') 'you must install boto: pip install boto')
if not self.s3conn: aws_access_key_id = None
try: aws_secret_access_key = None
self.s3conn = connect_s3()
except Exception: #pragma: no cover
self.s3conn = connect_s3(anon=True)
parts = urlsplit(url) parts = urlsplit(url)
bucket = self.s3conn.get_bucket(parts.netloc) if parts.username and parts.password:
aws_access_key_id = parts.username
aws_secret_access_key = parts.password
bucket_name = parts.netloc.split('@', 1)[-1]
else:
bucket_name = parts.netloc
headers = {'Range': BlockLoader._make_range_header(offset, length)} if not self.s3conn:
try:
self.s3conn = connect_s3(aws_access_key_id, aws_secret_access_key)
except Exception: #pragma: no cover
self.s3conn = connect_s3(anon=True)
bucket = self.s3conn.get_bucket(bucket_name)
key = bucket.get_key(parts.path) key = bucket.get_key(parts.path)
result = key.get_contents_as_string(headers=headers) if offset == 0 and length == -1:
key.close() headers = {}
else:
headers = {'Range': BlockLoader._make_range_header(offset, length)}
return BytesIO(result) # Read range
key.open_read(headers=headers)
return key
#================================================================= #=================================================================