mirror of
https://github.com/internetarchive/warcprox.git
synced 2025-01-18 13:22:09 +01:00
commit
c027659001
@ -437,8 +437,8 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
|
|||||||
else:
|
else:
|
||||||
if hash_plus_url in hash_plus_urls:
|
if hash_plus_url in hash_plus_urls:
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
'discarding duplicate and setting do_not_archive for %s, hash %s'.format(
|
'discarding duplicate and setting do_not_archive for %s, hash %s',
|
||||||
recorded_url.url, payload_hash))
|
recorded_url.url, payload_hash)
|
||||||
recorded_url.do_not_archive = True
|
recorded_url.do_not_archive = True
|
||||||
discards.append(payload_hash)
|
discards.append(payload_hash)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
|
Loading…
x
Reference in New Issue
Block a user