backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

try iso-8859-1

Browse Source
This commit is contained in:
Barbara Miller 2023-08-15 15:50:02 -07:00
parent 533f5c0af2
commit 887680b0ec
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warcprox/dedup.py
View File

@ -492,7 +492,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor, LimitRevisitsPGMix
hash_plus_url = b''.join((payload_hash, recorded_url.url))
if (recorded_url.response_recorder
and hash_plus_url not in hash_plus_urls
and not self.limit_revisits(recorded_url, hash_plus_url.decode())
and not self.limit_revisits(recorded_url, hash_plus_url.decode('iso-8859-1'))
and self.trough_dedup_db.should_dedup(recorded_url)):
hash_plus_urls.add(hash_plus_url)