From fc9d659b5d32beae6b2fc938eccb234e10b63a72 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 28 Mar 2015 16:41:52 -0700 Subject: [PATCH] loaders: switch BlockLoader to use requests instead of urliib2 --- pywb/utils/loaders.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pywb/utils/loaders.py b/pywb/utils/loaders.py index cc1eb77f..b5b39568 100644 --- a/pywb/utils/loaders.py +++ b/pywb/utils/loaders.py @@ -6,7 +6,8 @@ local and remote access import os import hmac import urllib -import urllib2 +#import urllib2 +import requests import urlparse import time import pkg_resources @@ -142,6 +143,7 @@ class BlockLoader(object): """ def __init__(self, cookie_maker=None): self.cookie_maker = cookie_maker + self.session = None def load(self, url, offset=0, length=-1): """ @@ -208,8 +210,13 @@ class BlockLoader(object): else: headers['Cookie'] = self.cookie_maker.make() - request = urllib2.Request(url, headers=headers) - return urllib2.urlopen(request) + if not self.session: + self.session = requests.Session() + + r = self.session.get(url, headers=headers, stream=True) + return r.raw + #request = urllib2.Request(url, headers=headers) + #return urllib2.urlopen(request) #=================================================================