From e74407591352ebb25a99c5c4a29229d0c47a6c4f Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 2 Dec 2021 11:46:30 -0800 Subject: [PATCH] python 3.5 version, mostly --- warcprox/dedup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 2648bf1..98cfea1 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -418,8 +418,8 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor): if (recorded_url.response_recorder and recorded_url.payload_digest and self.trough_dedup_db.should_dedup(recorded_url) - and f'{recorded_url.payload_digest}{recorded_url.url}' not in hash_plus_urls): - hash_plus_urls.add(f'{recorded_url.payload_digest}{recorded_url.url}') + and '{}{}'.format(recorded_url.payload_digest, recorded_url.url) not in hash_plus_urls): + hash_plus_urls.add('{}{}'.format(recorded_url.payload_digest, recorded_url.url)) if (recorded_url.warcprox_meta and 'dedup-buckets' in recorded_url.warcprox_meta): for bucket, bucket_mode in recorded_url.warcprox_meta["dedup-buckets"].items(): @@ -427,9 +427,9 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor): else: buckets['__unspecified__'].append(recorded_url) else: - if f'{recorded_url.payload_digest}{recorded_url.url}' in hash_plus_urls: + if recorded_url.payload_digest and '{}{}'.format(recorded_url.payload_digest, recorded_url.url) in hash_plus_urls: self.logger.debug( - f'discarding duplicate {recorded_url.payload_digest} {recorded_url.url}') + 'discarding duplicate {} {}'.format(recorded_url.payload_digest, recorded_url.url)) discards.append( warcprox.digest_str( recorded_url.payload_digest, self.options.base32)