From d035147e3ea8c04598f61c3ffe3f68332530f69b Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Sun, 24 Sep 2017 13:36:12 +0000 Subject: [PATCH 1/3] Remove redundant close method from DedupDb and RethinkDedupDb I'm trying to implement another DedupDb interface and I looked into the use of each method. The ``close`` method of ``dedup.DedupDb`` and ``deup.RethinkDedupDb`` is empty. It is also invoked from ``controller``. Since it doesn't do anything and it won't in the foreseeable future, let's remove it. --- warcprox/controller.py | 2 -- warcprox/dedup.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/warcprox/controller.py b/warcprox/controller.py index b3f3eaf..42f71de 100644 --- a/warcprox/controller.py +++ b/warcprox/controller.py @@ -211,8 +211,6 @@ class WarcproxController(object): if self.proxy.stats_db: self.proxy.stats_db.stop() - if any((t.dedup_db for t in self.warc_writer_threads)): - self.warc_writer_threads[0].dedup_db.close() self.proxy_thread.join() if self.playback_proxy is not None: diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 78c5434..7a7a025 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -58,9 +58,6 @@ class DedupDb(object): def stop(self): pass - def close(self): - pass - def sync(self): pass @@ -154,9 +151,6 @@ class RethinkDedupDb: def stop(self): pass - def close(self): - pass - def sync(self): pass From eb266f198df6531d19cbaa94cee458e6da4a131b Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Sun, 24 Sep 2017 13:44:13 +0000 Subject: [PATCH 2/3] Remove redundant stop() & sync() dedup methods Similarly with my previous commits, these methods do nothing. I think that the reason they are here is because the author uses the same style in other places in the code (e.g. ``warcprox.stats.StatsDb``). Similar methods exist there. --- warcprox/dedup.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 7a7a025..59c6040 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -55,12 +55,6 @@ class DedupDb(object): conn.commit() conn.close() - def stop(self): - pass - - def sync(self): - pass - def save(self, digest_key, response_record, bucket=""): record_id = response_record.get_header(warctools.WarcRecord.ID).decode('latin1') url = response_record.get_header(warctools.WarcRecord.URL).decode('latin1') @@ -148,12 +142,6 @@ class RethinkDedupDb: def start(self): pass - def stop(self): - pass - - def sync(self): - pass - def save(self, digest_key, response_record, bucket=""): k = digest_key.decode("utf-8") if isinstance(digest_key, bytes) else digest_key k = "{}|{}".format(k, bucket) From 4e7d8fa9173650793c41603c9b733907dbe5c168 Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Fri, 29 Sep 2017 06:36:37 +0000 Subject: [PATCH 3/3] Remove deleted ``close`` method call from test. --- tests/test_warcprox.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_warcprox.py b/tests/test_warcprox.py index dd80a86..40bdc24 100755 --- a/tests/test_warcprox.py +++ b/tests/test_warcprox.py @@ -274,12 +274,10 @@ def rethink_dedup_db(request, rethinkdb_servers, captures_db): ddb = warcprox.dedup.RethinkDedupDb(rr) def fin(): - if rethinkdb_servers: - ddb.close() - if not captures_db: - logging.info('dropping rethinkdb database {}'.format(db)) - result = ddb.rr.db_drop(db).run() - logging.info("result=%s", result) + if rethinkdb_servers and not captures_db: + logging.info('dropping rethinkdb database {}'.format(db)) + result = ddb.rr.db_drop(db).run() + logging.info("result=%s", result) request.addfinalizer(fin) return ddb