backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

recorded_url.do_not_archive = True

Browse Source
This commit is contained in:
Barbara Miller 2021-12-08 11:04:09 -08:00
parent da089e0a92
commit 7d4c8dcb4e
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
warcprox/dedup.py
View File

@ -435,7 +435,8 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
else: else:
if hash_plus_url in hash_plus_urls: if hash_plus_url in hash_plus_urls:
self.logger.debug( self.logger.debug(
'discarding duplicate {}'.format(hash_plus_url)) 'discarding duplicate {}, setting do_not_archive'.format(hash_plus_url))
recorded_url.do_not_archive = True
discards.append( discards.append(
warcprox.digest_str( warcprox.digest_str(
recorded_url.payload_digest, self.options.base32) recorded_url.payload_digest, self.options.base32)