backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

eh, don't prefix sqlite filenames with 'warcprox-trough-'; logging tweaks

Browse Source
This commit is contained in:
Noah Levitt 2017-11-10 13:37:09 -08:00
parent b2adb778ee
commit cdd747f48e
1 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
warcprox/dedup.py
View File

@ -280,7 +280,7 @@ class TroughClient(object):
lambda svc: r.now().sub( lambda svc: r.now().sub(
svc['last_heartbeat']).lt(svc['ttl']) svc['last_heartbeat']).lt(svc['ttl'])
).order_by('load') ).order_by('load')
logging.debug('querying rethinkdb: %r', reql) self.logger.debug('querying rethinkdb: %r', reql)
results = reql.run() results = reql.run()
if results: if results:
return results[0]['url'] return results[0]['url']
@ -329,20 +329,18 @@ class TroughDedupDb(object):
def _write_url(self, bucket): def _write_url(self, bucket):
if not bucket in self._write_url_cache: if not bucket in self._write_url_cache:
segment_id = 'warcprox-trough-%s' % bucket
self._write_url_cache[bucket] = self._trough_cli.write_url( self._write_url_cache[bucket] = self._trough_cli.write_url(
segment_id, self.SCHEMA_ID) bucket, self.SCHEMA_ID)
logging.info( self.logger.info(
'bucket %r write url is %r', bucket, 'trough dedup bucket %r write url is %r', bucket,
self._write_url_cache[bucket]) self._write_url_cache[bucket])
return self._write_url_cache[bucket] return self._write_url_cache[bucket]
def _read_url(self, bucket): def _read_url(self, bucket):
if not self._read_url_cache.get(bucket): if not self._read_url_cache.get(bucket):
segment_id = 'warcprox-trough-%s' % bucket self._read_url_cache[bucket] = self._trough_cli.read_url(bucket)
self._read_url_cache[bucket] = self._trough_cli.read_url(segment_id) self.logger.info(
logging.info( 'trough dedup bucket %r read url is %r', bucket,
'bucket %r read url is %r', bucket,
self._read_url_cache[bucket]) self._read_url_cache[bucket])
return self._read_url_cache[bucket] return self._read_url_cache[bucket]
@ -381,18 +379,18 @@ class TroughDedupDb(object):
try: try:
response = requests.post(write_url, sql) response = requests.post(write_url, sql)
except: except:
logging.error( self.logger.error(
'problem with trough write url %r', write_url, 'problem with trough write url %r', write_url,
exc_info=True) exc_info=True)
del self._write_url_cache[bucket] del self._write_url_cache[bucket]
return return
if response.status_code != 200: if response.status_code != 200:
del self._write_url_cache[bucket] del self._write_url_cache[bucket]
logging.warn( self.logger.warn(
'unexpected response %r %r %r to sql=%r', 'unexpected response %r %r %r to sql=%r',
response.status_code, response.reason, response.text, sql) response.status_code, response.reason, response.text, sql)
else: else:
logging.trace('posted %r to %s', sql, write_url) self.logger.debug('posted %r to %s', sql, write_url)
def lookup(self, digest_key, bucket='__unspecified__', url=None): def lookup(self, digest_key, bucket='__unspecified__', url=None):
read_url = self._read_url(bucket) read_url = self._read_url(bucket)
@ -403,17 +401,17 @@ class TroughDedupDb(object):
try: try:
response = requests.post(read_url, sql) response = requests.post(read_url, sql)
except: except:
logging.error( self.logger.error(
'problem with trough read url %r', read_url, exc_info=True) 'problem with trough read url %r', read_url, exc_info=True)
del self._read_url_cache[bucket] del self._read_url_cache[bucket]
return None return None
if response.status_code != 200: if response.status_code != 200:
del self._read_url_cache[bucket] del self._read_url_cache[bucket]
logging.warn( self.logger.warn(
'unexpected response %r %r %r to sql=%r', 'unexpected response %r %r %r to sql=%r',
response.status_code, response.reason, response.text, sql) response.status_code, response.reason, response.text, sql)
return None return None
logging.debug('got %r from query %r', response.text, sql) self.logger.trace('got %r from query %r', response.text, sql)
results = json.loads(response.text) results = json.loads(response.text)
assert len(results) <= 1 # sanity check (digest_key is primary key) assert len(results) <= 1 # sanity check (digest_key is primary key)
if results: if results: