mirror of
https://github.com/internetarchive/warcprox.git
synced 2025-01-18 13:22:09 +01:00
Merge branch 'WT-31' into qa
This commit is contained in:
commit
1d3e3b3671
@ -435,7 +435,8 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
|
||||
else:
|
||||
if hash_plus_url in hash_plus_urls:
|
||||
self.logger.debug(
|
||||
'discarding duplicate {}'.format(hash_plus_url))
|
||||
'discarding duplicate {}, setting do_not_archive'.format(hash_plus_url))
|
||||
recorded_url.do_not_archive = True
|
||||
discards.append(
|
||||
warcprox.digest_str(
|
||||
recorded_url.payload_digest, self.options.base32)
|
||||
|
Loading…
x
Reference in New Issue
Block a user