backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

fix typos

Browse Source
This commit is contained in:
Barbara Miller 2023-06-05 17:56:26 -07:00
parent 1dc7de7dd8
commit 419e5bc536
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/writerthread.py
View File

@ -117,7 +117,7 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
and "ait-job-id" in recorded_url.warcprox_meta["metadata"] and "ait-job-id" in recorded_url.warcprox_meta["metadata"]
): ):
crawl_id = recorded_url.warcprox_meta["metadata"]["ait-job-id"] crawl_id = recorded_url.warcprox_meta["metadata"]["ait-job-id"]
if recorded_url.payload_digest in revisits[crawl_id]: if recorded_url.payload_digest in self.revisits[crawl_id]:
self.logger.info( self.logger.info(
"Found duplicate revisit, skipping: %s, hash: %s", "Found duplicate revisit, skipping: %s, hash: %s",
recorded_url.url, recorded_url.url,
@ -125,7 +125,7 @@ class WarcWriterProcessor(warcprox.BaseStandardPostfetchProcessor):
) )
return True return True
else: else:
revisits[crawl_id].add(recorded_url.payload_digest) self.revisits[crawl_id].add(recorded_url.payload_digest)
return False return False