backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

on error from trough read or write url, delete read/write url from cache, so next request will retrieve a fresh, hopefully working, url (n.b. not covered by automated tests at this point)

Browse Source
This commit is contained in:
Noah Levitt 2017-11-03 14:16:09 -07:00
parent 147b097a53
commit 3dbfc06e68
1 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
warcprox/dedup.py
View File

@ -256,7 +256,6 @@ class TroughClient(object):
self.svcreg = doublethink.ServiceRegistry(self.rr) self.svcreg = doublethink.ServiceRegistry(self.rr)
def segment_manager_url(self): def segment_manager_url(self):
# XXX cache until expired (check last_heartbeat and ttl)
master_node = self.svcreg.unique_service('trough-sync-master') master_node = self.svcreg.unique_service('trough-sync-master')
assert master_node assert master_node
return master_node['url'] return master_node['url']
@ -379,8 +378,16 @@ class TroughDedupDb(object):
'values (%s, %s, %s, %s);') % ( 'values (%s, %s, %s, %s);') % (
self.sql_value(digest_key), self.sql_value(url), self.sql_value(digest_key), self.sql_value(url),
self.sql_value(warc_date), self.sql_value(record_id)) self.sql_value(warc_date), self.sql_value(record_id))
response = requests.post(write_url, sql) try:
response = requests.post(write_url, sql)
except:
logging.error(
'problem with trough write url %r', write_url,
exc_info=True)
del self._write_url_cache[bucket]
return
if response.status_code != 200: if response.status_code != 200:
del self._write_url_cache[bucket]
logging.warn( logging.warn(
'unexpected response %r %r %r to sql=%r', 'unexpected response %r %r %r to sql=%r',
response.status_code, response.reason, response.text, sql) response.status_code, response.reason, response.text, sql)
@ -393,8 +400,15 @@ class TroughDedupDb(object):
return None return None
sql = 'select * from dedup where digest_key=%s;' % ( sql = 'select * from dedup where digest_key=%s;' % (
self.sql_value(digest_key)) self.sql_value(digest_key))
response = requests.post(read_url, sql) try:
response = requests.post(read_url, sql)
except:
logging.error(
'problem with trough read url %r', read_url, exc_info=True)
del self._read_url_cache[bucket]
return None
if response.status_code != 200: if response.status_code != 200:
del self._read_url_cache[bucket]
logging.warn( logging.warn(
'unexpected response %r %r %r to sql=%r', 'unexpected response %r %r %r to sql=%r',
response.status_code, response.reason, response.text, sql) response.status_code, response.reason, response.text, sql)