From b91a7d1d8932fb75f0ae1f0726409f8c2904e3cd Mon Sep 17 00:00:00 2001
From: Barbara Miller <barbara@archive.org>
Date: Wed, 28 Jun 2023 11:43:53 -0700
Subject: [PATCH] more updates qa prototyping

---
 warcprox/dedup.py | 45 ++++++++++++++++++++++-----------------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/warcprox/dedup.py b/warcprox/dedup.py
index 93a335b..c49edd4 100644
--- a/warcprox/dedup.py
+++ b/warcprox/dedup.py
@@ -62,21 +62,21 @@ class DedupableMixin(object):
         else:
             return recorded_url.response_recorder.payload_size() > self.min_binary_size
 
-class LimitRevisitsPGMixin(object):
+class LimitRevisitsPGMixin():
     """
     Limit revisits recorded to one per revisit_key
     """
-    def __init__(self, options=warcprox.Options()):
+    def __init__(self):
         self.datasource = "postgresql://archiveit@db.qa-archive-it.org/archiveit" # "postgresql://user@db_host/db_name"
         self.datatable  = "crawl_revisits"  # postgres table in db_name
 
     def limit_revisits(self, recorded_url, hash_plus_url=None, revisit_key=None):
+        # tracks revisits, returns True when we've seen revisit before, else False
         if not hash_plus_url:
-            hash_plus_url = b"".join(
-                (warcprox.digest_str(recorded_url.payload_digest,
-                                     self.options.base32),
-                 recorded_url.url)
-            ).decode()
+            digest = warcprox.digest_str(recorded_url.payload_digest,
+                                     self.options.base32)
+            digest = digest[5:] if digest.startswith(b'sha1:') else digest
+            hash_plus_url = b"".join([digest, recorded_url.url]).decode()
         if not revisit_key:
             # use ait-job-id if available
             if (
@@ -84,11 +84,11 @@ class LimitRevisitsPGMixin(object):
                 and "metadata" in recorded_url.warcprox_meta
                 and "ait-job-id" in recorded_url.warcprox_meta["metadata"]
             ):
-                revisit_key = recorded_url.warcprox_meta["metadata"]["ait-job-id"]
+                revisit_key = str(recorded_url.warcprox_meta["metadata"]["ait-job-id"])
             else:
-                revisit_key = 'all'
+                revisit_key = '__unspecified__'
 
-        query = "SELECT exists(SELECT 1 FROM crawl_revisits WHERE hash_plus_url = %s LIMIT 1);"
+        query = "SELECT exists(SELECT 1 FROM crawl_revisits WHERE hash_plus_url = %s and crawl_id = %s LIMIT 1);"
 
         try:
             conn = psycopg2.connect(self.datasource)
@@ -97,23 +97,22 @@ class LimitRevisitsPGMixin(object):
             return False
         cur = conn.cursor()
         try:
-            cur.execute(query, (hash_plus_url,))
+            cur.execute(query, (hash_plus_url, revisit_key))
         except Exception as e:
-            self.logger.warning("exception querying for %s in %s: %s", hash_plus_url, revisit_key, e)
+            self.logger.warning("exception querying for %s in %s: %s", digest, revisit_key, e)
             return False
         result = cur.fetchone()
-        if result[0]:
-            logging.info("result[0]: %s", result[0])
 
-        if result[0] and result[0] == True:
-            logging.info("skipping revisit for url %s and hash %s", recorded_url.url, hash)
+        if result and result == (True, ):
+            logging.info("skipping revisit for url %s and hash %s", recorded_url.url, digest)
             return True
-        else:
-            query = "INSERT INTO crawl_revisits (crawl_id, hash_plus_url) VALUES (%s, %s);"
-            try:
-                cur.execute(query, (revisit_key, hash_plus_url))
-            except Exception as e:
-                self.logger.warning("exception inserting %s in %s: %s", hash_plus_url, revisit_key, e)
+
+        query = "INSERT INTO crawl_revisits (crawl_id, hash_plus_url) VALUES (%s, %s);"
+        try:
+            cur.execute(query, (revisit_key, hash_plus_url))
+            conn.commit()
+        except Exception as e:
+            self.logger.warning("exception inserting %s in %s for %s: %s", digest, revisit_key, recorded_url.url, e)
 
         return False
 
@@ -516,7 +515,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor, LimitRevisitsPGMix
 
     def __init__(self, trough_dedup_db, options=warcprox.Options()):
         warcprox.BaseBatchPostfetchProcessor.__init__(self, options)
-        LimitRevisitsPGMixin.__init__(self, options)
+        LimitRevisitsPGMixin.__init__(self)
         self.trough_dedup_db = trough_dedup_db
 
     def _startup(self):