backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

indexsource: use logging for failure reports

Browse Source 
don't add connection: close by default now that better pooling is in place
This commit is contained in:
Ilya Kreymer 2017-07-02 17:09:01 +00:00
parent 84eb070938
commit f3487a1922
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pywb/warcserver/index/indexsource.py
View File

@ -16,12 +16,15 @@ import redis
import requests import requests
import re import re
import logging
#============================================================================= #=============================================================================
class BaseIndexSource(object): class BaseIndexSource(object):
WAYBACK_ORIG_SUFFIX = '{timestamp}id_/{url}' WAYBACK_ORIG_SUFFIX = '{timestamp}id_/{url}'
logger = logging.getLogger(__name__)
def load_index(self, params): #pragma: no cover def load_index(self, params): #pragma: no cover
raise NotImplemented() raise NotImplemented()
@ -119,7 +122,7 @@ class RemoteIndexSource(BaseIndexSource):
r = self.sesh.get(api_url, timeout=params.get('_timeout')) r = self.sesh.get(api_url, timeout=params.get('_timeout'))
r.raise_for_status() r.raise_for_status()
except Exception as e: except Exception as e:
print('FAILED: ' + str(e)) self.logger.debug('FAILED: ' + str(e))
raise NotFoundException(api_url) raise NotFoundException(api_url)
lines = r.content.strip().split(b'\n') lines = r.content.strip().split(b'\n')
@ -387,7 +390,7 @@ class MementoIndexSource(BaseIndexSource):
res = self.sesh.head(url, headers=headers) res = self.sesh.head(url, headers=headers)
res.raise_for_status() res.raise_for_status()
except Exception as e: except Exception as e:
print('FAILED: ' + str(e)) self.logger.debug('FAILED: ' + str(e))
raise NotFoundException(url) raise NotFoundException(url)
links = res.headers.get('Link') links = res.headers.get('Link')
@ -398,8 +401,7 @@ class MementoIndexSource(BaseIndexSource):
return links return links
def _get_headers(self, params): def _get_headers(self, params):
headers = {'Connection': 'close'} return {}
return headers
def handle_timemap(self, params): def handle_timemap(self, params):
url = res_template(self.timemap_url, params) url = res_template(self.timemap_url, params)
@ -413,7 +415,7 @@ class MementoIndexSource(BaseIndexSource):
assert(res.text) assert(res.text)
except Exception as e: except Exception as e:
print('FAILED: ' + str(e)) self.logger.debug('FAILED: ' + str(e))
raise NotFoundException(url) raise NotFoundException(url)
links = res.text links = res.text