mirror of
https://github.com/internetarchive/warcprox.git
synced 2025-01-18 13:22:09 +01:00
limit_revisits wants str, not bytes
This commit is contained in:
parent
a86169c56c
commit
533f5c0af2
@ -492,7 +492,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor, LimitRevisitsPGMix
|
||||
hash_plus_url = b''.join((payload_hash, recorded_url.url))
|
||||
if (recorded_url.response_recorder
|
||||
and hash_plus_url not in hash_plus_urls
|
||||
and not self.limit_revisits(recorded_url, hash_plus_url)
|
||||
and not self.limit_revisits(recorded_url, hash_plus_url.decode())
|
||||
and self.trough_dedup_db.should_dedup(recorded_url)):
|
||||
hash_plus_urls.add(hash_plus_url)
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user