backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'limit_revisits' into qa

Browse Source
This commit is contained in:
Barbara Miller 2023-06-28 17:35:58 -07:00
commit 8563b95ff6
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
warcprox/dedup.py
View File

@ -66,7 +66,6 @@ class LimitRevisitsPGMixin():
"""
Limit revisits recorded to one per revisit_key
"""
def __init__(self):
self.datasource = "postgresql://archiveit@db.qa-archive-it.org/archiveit" # "postgresql://user@db_host/db_name"
self.datatable = "crawl_revisits" # postgres table in db_name
@ -85,7 +84,7 @@ class LimitRevisitsPGMixin():
and "metadata" in recorded_url.warcprox_meta
and "ait-job-id" in recorded_url.warcprox_meta["metadata"]
):
revisit_key = recorded_url.warcprox_meta["metadata"]["ait-job-id"]
revisit_key = str(recorded_url.warcprox_meta["metadata"]["ait-job-id"])
else:
revisit_key = '__unspecified__'