diff --git a/warcprox/bigtable.py b/warcprox/bigtable.py index a1620e2..3c610e4 100644 --- a/warcprox/bigtable.py +++ b/warcprox/bigtable.py @@ -14,11 +14,11 @@ import hashlib class RethinkCaptures: logger = logging.getLogger("warcprox.bigtables.RethinkCaptures") - def __init__(self, r, table="captures", shards=3, replicas=3, options=warcprox.Options()): + def __init__(self, r, table="captures", shards=None, replicas=None, options=warcprox.Options()): self.r = r self.table = table - self.shards = shards - self.replicas = replicas + self.shards = shards or len(r.servers) + self.replicas = replicas or min(3, len(r.servers)) self.options = options self._ensure_db_table() diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 33e93af..358af05 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -83,11 +83,11 @@ def decorate_with_dedup_info(dedup_db, recorded_url, base32=False): class RethinkDedupDb: logger = logging.getLogger("warcprox.dedup.RethinkDedupDb") - def __init__(self, r, table="dedup", shards=3, replicas=3, options=warcprox.Options()): + def __init__(self, r, table="dedup", shards=None, replicas=None, options=warcprox.Options()): self.r = r self.table = table - self.shards = shards - self.replicas = replicas + self.shards = shards or len(r.servers) + self.replicas = replicas or min(3, len(r.servers)) self._ensure_db_table() self.options = options @@ -98,7 +98,8 @@ class RethinkDedupDb: self.r.db_create(self.r.db).run() tables = self.r.table_list().run() if not self.table in tables: - self.logger.info("creating rethinkdb table %s in database %s", repr(self.table), repr(self.r.db)) + self.logger.info("creating rethinkdb table %s in database %s shards=%s replicas=%s", + repr(self.table), repr(self.r.db), self.shards, self.replicas) self.r.table_create(self.table, primary_key="key", shards=self.shards, replicas=self.replicas).run() def close(self): diff --git a/warcprox/stats.py b/warcprox/stats.py index da394a7..a6e5dbf 100644 --- a/warcprox/stats.py +++ b/warcprox/stats.py @@ -112,11 +112,11 @@ class StatsDb: class RethinkStatsDb: logger = logging.getLogger("warcprox.stats.RethinkStatsDb") - def __init__(self, r, table="stats", shards=3, replicas=3, options=warcprox.Options()): + def __init__(self, r, table="stats", shards=None, replicas=None, options=warcprox.Options()): self.r = r self.table = table - self.shards = shards - self.replicas = replicas + self.shards = shards or len(r.servers) + self.replicas = replicas or min(3, len(r.servers)) self._ensure_db_table() self.options = options @@ -127,7 +127,8 @@ class RethinkStatsDb: self.r.db_create(self.r.db).run() tables = self.r.table_list().run() if not self.table in tables: - self.logger.info("creating rethinkdb table %s in database %s", repr(self.table), repr(self.r.db)) + self.logger.info("creating rethinkdb table %s in database %s shards=%s replicas=%s", + repr(self.table), repr(self.r.db), self.shards, self.replicas) self.r.table_create(self.table, primary_key="bucket", shards=self.shards, replicas=self.replicas).run() def close(self):