backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

fixes for qa prototyping

Browse Source
This commit is contained in:
Barbara Miller 2023-06-27 11:47:55 -07:00
parent d9145eefb5
commit b3f7b09298
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
warcprox/dedup.py
View File

@ -65,11 +65,11 @@ class LimitRevisitsPGMixin(object):
""" """
Limit revisits recorded to one per revisit_key Limit revisits recorded to one per revisit_key
""" """
def __init__(self, datasource, options=warcprox.Options()): def __init__(self, options=warcprox.Options()):
import psycopg2 import psycopg2
from psycopg2 import extras # needed from psycopg2 import extras # TODO: needed?
self.datasource = datasource # "postgresql://user@db_host/db_name" self.datasource = "postgresql://archiveit@db.qa-archive-it.org/archiveit" # "postgresql://user@db_host/db_name"
self.datatable = 'crawled_urls' # postgres table in db_name self.datatable = "crawl_revisits" # postgres table in db_name
def limit_revisits(self, recorded_url, hash_plus_url=None, revisit_key=None): def limit_revisits(self, recorded_url, hash_plus_url=None, revisit_key=None):
if not hash_plus_url: if not hash_plus_url:
@ -78,7 +78,6 @@ class LimitRevisitsPGMixin(object):
self.options.base32), self.options.base32),
recorded_url.url) recorded_url.url)
) )
if not revisit_key: if not revisit_key:
# use ait-job-id if available # use ait-job-id if available
if ( if (
@ -90,7 +89,7 @@ class LimitRevisitsPGMixin(object):
else: else:
revisit_key = 'all' revisit_key = 'all'
query = f"SELECT exists(SELECT 1 FROM {self.datatable} WHERE hash_plus_url = {hash_plus_url} LIMIT 1;" query = f"SELECT exists(SELECT 1 FROM {self.datatable} WHERE hash_plus_url = {hash_plus_url} LIMIT 1);"
try: try:
conn = psycopg2.connect(self.datasource) conn = psycopg2.connect(self.datasource)
@ -104,12 +103,14 @@ class LimitRevisitsPGMixin(object):
self.logger.warning("exception querying for %s in %s: %s", hash_plus_url, revisit_key, e) self.logger.warning("exception querying for %s in %s: %s", hash_plus_url, revisit_key, e)
return False return False
result = cur.fetchone() result = cur.fetchone()
if result[0]: if result[0]:
logging.info("result[0]: %s", result[0])
if result[0] and result[0] == True:
logging.info("skipping revisit for url %s and hash %s", recorded_url.url, hash) logging.info("skipping revisit for url %s and hash %s", recorded_url.url, hash)
return True return True
else: else:
query = "INSERT INTO {self.datable} VALUES(revisit_key, hash_plus_url);" query = f"INSERT INTO {self.datatable} VALUES({revisit_key}, {hash_plus_url});"
try: try:
cur.execute(query) cur.execute(query)
except Exception as e: except Exception as e:
@ -518,8 +519,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor, LimitRevisitsPGMix
def __init__(self, trough_dedup_db, options=warcprox.Options()): def __init__(self, trough_dedup_db, options=warcprox.Options()):
warcprox.BaseBatchPostfetchProcessor.__init__(self, options) warcprox.BaseBatchPostfetchProcessor.__init__(self, options)
LimitRevisitsPGMixin.__init__(self, datasource="", options) LimitRevisitsPGMixin.__init__(self, datasource, options)
# TODO: get datasource
self.trough_dedup_db = trough_dedup_db self.trough_dedup_db = trough_dedup_db
def _startup(self): def _startup(self):