From 1476bfec8c71854ad92083295270240fc4e7d93d Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 2 Dec 2021 11:09:17 -0800 Subject: [PATCH] discard batch hash+url match --- warcprox/dedup.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 77bc7c6..2648bf1 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -1,7 +1,7 @@ ''' warcprox/dedup.py - identical payload digest deduplication using sqlite db -Copyright (C) 2013-2018 Internet Archive +Copyright (C) 2013-2021 Internet Archive This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -412,10 +412,14 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor): ''' buckets = collections.defaultdict(list) discards = [] + # for duplicate checks, see https://webarchive.jira.com/browse/WT-31 + hash_plus_urls = set() for recorded_url in batch: if (recorded_url.response_recorder and recorded_url.payload_digest - and self.trough_dedup_db.should_dedup(recorded_url)): + and self.trough_dedup_db.should_dedup(recorded_url) + and f'{recorded_url.payload_digest}{recorded_url.url}' not in hash_plus_urls): + hash_plus_urls.add(f'{recorded_url.payload_digest}{recorded_url.url}') if (recorded_url.warcprox_meta and 'dedup-buckets' in recorded_url.warcprox_meta): for bucket, bucket_mode in recorded_url.warcprox_meta["dedup-buckets"].items(): @@ -423,6 +427,9 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor): else: buckets['__unspecified__'].append(recorded_url) else: + if f'{recorded_url.payload_digest}{recorded_url.url}' in hash_plus_urls: + self.logger.debug( + f'discarding duplicate {recorded_url.payload_digest} {recorded_url.url}') discards.append( warcprox.digest_str( recorded_url.payload_digest, self.options.base32)