backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

fix NameError, quiet logging

Browse Source
This commit is contained in:
Noah Levitt 2015-08-28 00:36:21 +00:00
parent 022f6e7215
commit a9986e4ce3
3 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warcprox/__init__.py
View File

@ -48,7 +48,7 @@ class Rethinker:
with self._random_server_connection() as conn: with self._random_server_connection() as conn:
try: try:
return query.run(conn, db=self.db) return query.run(conn, db=self.db)
except (ReqlAvailabilityError, ReqlTimeoutError) as e: except (r.ReqlAvailabilityError, r.ReqlTimeoutError) as e:
self.logger.error("will retry rethinkdb query/operation %s which failed like so:", exc_info=True) self.logger.error("will retry rethinkdb query/operation %s which failed like so:", exc_info=True)
version_bytes = _read_version_bytes().strip() version_bytes = _read_version_bytes().strip()

8
warcprox/bigtable.py
View File

@ -13,7 +13,7 @@ import surt
import os import os
class RethinkCaptures: class RethinkCaptures:
logger = logging.getLogger("warcprox.dedup.RethinkCaptures") logger = logging.getLogger("warcprox.bigtables.RethinkCaptures")
def __init__(self, servers=["localhost"], db="warcprox", table="captures", shards=3, replicas=3, options=warcprox.Options()): def __init__(self, servers=["localhost"], db="warcprox", table="captures", shards=3, replicas=3, options=warcprox.Options()):
self.r = warcprox.Rethinker(servers, db) self.r = warcprox.Rethinker(servers, db)
@ -47,7 +47,7 @@ class RethinkCaptures:
result = results[0] result = results[0]
else: else:
result = None result = None
self.logger.info("returning %s for sha1base32=%s", result, sha1base32) self.logger.debug("returning %s for sha1base32=%s bucket=%s", result, sha1base32, bucket)
return result return result
def notify(self, recorded_url, records): def notify(self, recorded_url, records):
@ -84,7 +84,7 @@ class RethinkCaptures:
result = self.r.run(r.table(self.table).insert(entry)) result = self.r.run(r.table(self.table).insert(entry))
if result["inserted"] == 1 and sorted(result.values()) != [0,0,0,0,0,1]: if result["inserted"] == 1 and sorted(result.values()) != [0,0,0,0,0,1]:
raise Exception("unexpected result %s saving %s", result, entry) raise Exception("unexpected result %s saving %s", result, entry)
self.logger.info("big capture table db saved %s", entry) self.logger.debug("big capture table db saved %s", entry)
class RethinkCapturesDedup: class RethinkCapturesDedup:
logger = logging.getLogger("warcprox.dedup.RethinkCapturesDedup") logger = logging.getLogger("warcprox.dedup.RethinkCapturesDedup")
@ -96,7 +96,6 @@ class RethinkCapturesDedup:
def lookup(self, digest_key, bucket="__unspecified__"): def lookup(self, digest_key, bucket="__unspecified__"):
k = digest_key.decode("utf-8") if isinstance(digest_key, bytes) else digest_key k = digest_key.decode("utf-8") if isinstance(digest_key, bytes) else digest_key
algo, value_str = k.split(":") algo, value_str = k.split(":")
self.logger.info("(algo,value_str)=(%s,%s)", algo, value_str)
if self.options.base32: if self.options.base32:
raw_digest = base64.b32decode(value_str, casefold=True) raw_digest = base64.b32decode(value_str, casefold=True)
else: else:
@ -104,7 +103,6 @@ class RethinkCapturesDedup:
entry = self.captures_db.find_response_by_digest(algo, raw_digest, bucket) entry = self.captures_db.find_response_by_digest(algo, raw_digest, bucket)
if entry: if entry:
dedup_info = {"url":entry["url"].encode("utf-8"), "date":entry["timestamp"].encode("utf-8"), "id":entry["warc_id"].encode("utf-8")} dedup_info = {"url":entry["url"].encode("utf-8"), "date":entry["timestamp"].encode("utf-8"), "id":entry["warc_id"].encode("utf-8")}
self.logger.info("returning %s for digest_key=%s", dedup_info, digest_key)
return dedup_info return dedup_info
else: else:
return None return None

2
warcprox/stats.py
View File

@ -1,5 +1,3 @@
# vim:set sw=4 et:
from __future__ import absolute_import from __future__ import absolute_import
try: try: