backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

avoid attempting to create tables with more shards or replicas than the number of servers

Browse Source
This commit is contained in:
Noah Levitt 2015-09-22 20:45:09 +00:00
parent 0171cdd01d
commit 69d641cd50
3 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
warcprox/bigtable.py
View File

@ -14,11 +14,11 @@ import hashlib
class RethinkCaptures:
logger = logging.getLogger("warcprox.bigtables.RethinkCaptures")
def __init__(self, r, table="captures", shards=3, replicas=3, options=warcprox.Options()):
def __init__(self, r, table="captures", shards=None, replicas=None, options=warcprox.Options()):
self.r = r
self.table = table
self.shards = shards
self.replicas = replicas
self.shards = shards or len(r.servers)
self.replicas = replicas or min(3, len(r.servers))
self.options = options
self._ensure_db_table()

9
warcprox/dedup.py
View File

@ -83,11 +83,11 @@ def decorate_with_dedup_info(dedup_db, recorded_url, base32=False):
class RethinkDedupDb:
logger = logging.getLogger("warcprox.dedup.RethinkDedupDb")
def __init__(self, r, table="dedup", shards=3, replicas=3, options=warcprox.Options()):
def __init__(self, r, table="dedup", shards=None, replicas=None, options=warcprox.Options()):
self.r = r
self.table = table
self.shards = shards
self.replicas = replicas
self.shards = shards or len(r.servers)
self.replicas = replicas or min(3, len(r.servers))
self._ensure_db_table()
self.options = options
@ -98,7 +98,8 @@ class RethinkDedupDb:
self.r.db_create(self.r.db).run()
tables = self.r.table_list().run()
if not self.table in tables:
self.logger.info("creating rethinkdb table %s in database %s", repr(self.table), repr(self.r.db))
self.logger.info("creating rethinkdb table %s in database %s shards=%s replicas=%s",
repr(self.table), repr(self.r.db), self.shards, self.replicas)
self.r.table_create(self.table, primary_key="key", shards=self.shards, replicas=self.replicas).run()
def close(self):

9
warcprox/stats.py
View File

@ -112,11 +112,11 @@ class StatsDb:
class RethinkStatsDb:
logger = logging.getLogger("warcprox.stats.RethinkStatsDb")
def __init__(self, r, table="stats", shards=3, replicas=3, options=warcprox.Options()):
def __init__(self, r, table="stats", shards=None, replicas=None, options=warcprox.Options()):
self.r = r
self.table = table
self.shards = shards
self.replicas = replicas
self.shards = shards or len(r.servers)
self.replicas = replicas or min(3, len(r.servers))
self._ensure_db_table()
self.options = options
@ -127,7 +127,8 @@ class RethinkStatsDb:
self.r.db_create(self.r.db).run()
tables = self.r.table_list().run()
if not self.table in tables:
self.logger.info("creating rethinkdb table %s in database %s", repr(self.table), repr(self.r.db))
self.logger.info("creating rethinkdb table %s in database %s shards=%s replicas=%s",
repr(self.table), repr(self.r.db), self.shards, self.replicas)
self.r.table_create(self.table, primary_key="bucket", shards=self.shards, replicas=self.replicas).run()
def close(self):