backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

initial deploy fixes

Browse Source
This commit is contained in:
Barbara Miller 2023-07-12 17:56:39 -07:00
parent af4c8b071a
commit 548c4e5cab
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/dedup.py
View File

@ -120,7 +120,7 @@ class LimitRevisitsPGMixin():
revisit_key = str(recorded_url.warcprox_meta["metadata"]["ait-job-id"]) revisit_key = str(recorded_url.warcprox_meta["metadata"]["ait-job-id"])
else: else:
revisit_key = '__unspecified__' revisit_key = '__unspecified__'
return skip_revisit(hash_plus_url, revisit_key, self.conn) return skip_revisit(hash_plus_url, revisit_key, self._conn)
class DedupLoader(warcprox.BaseStandardPostfetchProcessor, DedupableMixin): class DedupLoader(warcprox.BaseStandardPostfetchProcessor, DedupableMixin):
@ -554,7 +554,7 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor, LimitRevisitsPGMix
recorded_url.dedup_info = entry recorded_url.dedup_info = entry
if recorded_url.dedup_info: if recorded_url.dedup_info:
recorded_url.do_not_archive = self.limit_revisits(recorded_url) recorded_url.do_not_archive = self.limit_revisits(recorded_url)
logging.info('%s', self.limit_revisits.cache_info()) logging.info('%s', skip_revisit.cache_info())
except Exception as e: except Exception as e:
# batch_lookup raised exception or something # batch_lookup raised exception or something
logging.warning( logging.warning(