backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

loaders: switch BlockLoader to use requests instead of urliib2

Browse Source
This commit is contained in:
Ilya Kreymer 2015-03-28 16:41:52 -07:00
parent f3a066f58b
commit fc9d659b5d
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pywb/utils/loaders.py
View File

@ -6,7 +6,8 @@ local and remote access
import os
import hmac
import urllib
import urllib2
#import urllib2
import requests
import urlparse
import time
import pkg_resources
@ -142,6 +143,7 @@ class BlockLoader(object):
"""
def __init__(self, cookie_maker=None):
self.cookie_maker = cookie_maker
self.session = None
def load(self, url, offset=0, length=-1):
"""
@ -208,8 +210,13 @@ class BlockLoader(object):
else:
headers['Cookie'] = self.cookie_maker.make()
request = urllib2.Request(url, headers=headers)
return urllib2.urlopen(request)
if not self.session:
self.session = requests.Session()
r = self.session.get(url, headers=headers, stream=True)
return r.raw
#request = urllib2.Request(url, headers=headers)
#return urllib2.urlopen(request)
#=================================================================