backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge pull request #35 from vbanos/dedup-redundant-code

Browse Source 
Remove redundant methods from dedup classes
This commit is contained in:
Noah Levitt 2017-09-29 11:42:47 -07:00 committed by GitHub
commit 0de10791aa
3 changed files with 4 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_warcprox.py
View File

@ -274,12 +274,10 @@ def rethink_dedup_db(request, rethinkdb_servers, captures_db):
ddb = warcprox.dedup.RethinkDedupDb(rr) ddb = warcprox.dedup.RethinkDedupDb(rr)
def fin(): def fin():
if rethinkdb_servers: if rethinkdb_servers and not captures_db:
ddb.close() logging.info('dropping rethinkdb database {}'.format(db))
if not captures_db: result = ddb.rr.db_drop(db).run()
logging.info('dropping rethinkdb database {}'.format(db)) logging.info("result=%s", result)
result = ddb.rr.db_drop(db).run()
logging.info("result=%s", result)
request.addfinalizer(fin) request.addfinalizer(fin)
return ddb return ddb

2
warcprox/controller.py
View File

@ -211,8 +211,6 @@ class WarcproxController(object):
if self.proxy.stats_db: if self.proxy.stats_db:
self.proxy.stats_db.stop() self.proxy.stats_db.stop()
if any((t.dedup_db for t in self.warc_writer_threads)):
self.warc_writer_threads[0].dedup_db.close()
self.proxy_thread.join() self.proxy_thread.join()
if self.playback_proxy is not None: if self.playback_proxy is not None:

18
warcprox/dedup.py
View File

@ -54,15 +54,6 @@ class DedupDb(object):
conn.commit() conn.commit()
conn.close() conn.close()
def stop(self):
pass
def close(self):
pass
def sync(self):
pass
def save(self, digest_key, response_record, bucket=""): def save(self, digest_key, response_record, bucket=""):
record_id = response_record.get_header(warctools.WarcRecord.ID).decode('latin1') record_id = response_record.get_header(warctools.WarcRecord.ID).decode('latin1')
url = response_record.get_header(warctools.WarcRecord.URL).decode('latin1') url = response_record.get_header(warctools.WarcRecord.URL).decode('latin1')
@ -150,15 +141,6 @@ class RethinkDedupDb:
def start(self): def start(self):
pass pass
def stop(self):
pass
def close(self):
pass
def sync(self):
pass
def save(self, digest_key, response_record, bucket=""): def save(self, digest_key, response_record, bucket=""):
k = digest_key.decode("utf-8") if isinstance(digest_key, bytes) else digest_key k = digest_key.decode("utf-8") if isinstance(digest_key, bytes) else digest_key
k = "{}|{}".format(k, bucket) k = "{}|{}".format(k, bucket)