backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

more hash_plus_url

Browse Source
This commit is contained in:
Barbara Miller 2021-12-06 19:43:27 -08:00
parent 5e5a74f204
commit 3eeccd0016
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
warcprox/dedup.py
View File

@ -424,7 +424,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
if (recorded_url.response_recorder if (recorded_url.response_recorder
and recorded_url.payload_digest and recorded_url.payload_digest
and self.trough_dedup_db.should_dedup(recorded_url) and self.trough_dedup_db.should_dedup(recorded_url)
and '{}{}'.format(recorded_url.payload_digest, recorded_url.url) not in hash_plus_urls): and hash_plus_url not in hash_plus_urls):
hash_plus_urls.add(hash_plus_url) hash_plus_urls.add(hash_plus_url)
if (recorded_url.warcprox_meta if (recorded_url.warcprox_meta
and 'dedup-buckets' in recorded_url.warcprox_meta): and 'dedup-buckets' in recorded_url.warcprox_meta):
@ -435,13 +435,13 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
else: else:
if hash_plus_url in hash_plus_urls: if hash_plus_url in hash_plus_urls:
self.logger.debug( self.logger.debug(
'discarding duplicate {}'.format(hash_plus_url) 'discarding duplicate {}'.format(hash_plus_url))
discards.append( discards.append(
warcprox.digest_str( warcprox.digest_str(
recorded_url.payload_digest, self.options.base32) recorded_url.payload_digest, self.options.base32)
if recorded_url.payload_digest else 'n/a') if recorded_url.payload_digest else 'n/a')
self.logger.debug( self.logger.debug(
'hash_plus_urls: {}'.format(hash_plus_urls)) 'hash_plus_urls: {}...'.format(hash_plus_urls[0]))
self.logger.debug( self.logger.debug(
'len(batch)=%s len(discards)=%s buckets=%s', 'len(batch)=%s len(discards)=%s buckets=%s',
len(batch), len(discards), len(batch), len(discards),