backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

loaders: switch BlockLoader to use requests instead of urliib2

Browse Source
This commit is contained in:
Ilya Kreymer 2015-03-28 16:41:52 -07:00
parent f3a066f58b
commit fc9d659b5d
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pywb/utils/loaders.py
View File

@ -6,7 +6,8 @@ local and remote access
import os import os
import hmac import hmac
import urllib import urllib
import urllib2 #import urllib2
import requests
import urlparse import urlparse
import time import time
import pkg_resources import pkg_resources
@ -142,6 +143,7 @@ class BlockLoader(object):
""" """
def __init__(self, cookie_maker=None): def __init__(self, cookie_maker=None):
self.cookie_maker = cookie_maker self.cookie_maker = cookie_maker
self.session = None
def load(self, url, offset=0, length=-1): def load(self, url, offset=0, length=-1):
""" """
@ -208,8 +210,13 @@ class BlockLoader(object):
else: else:
headers['Cookie'] = self.cookie_maker.make() headers['Cookie'] = self.cookie_maker.make()
request = urllib2.Request(url, headers=headers) if not self.session:
return urllib2.urlopen(request) self.session = requests.Session()
r = self.session.get(url, headers=headers, stream=True)
return r.raw
#request = urllib2.Request(url, headers=headers)
#return urllib2.urlopen(request)
#================================================================= #=================================================================