backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

log LRU cache info every 1000 requests

Browse Source 
to avoid writing to the log too often.
This commit is contained in:
Vangelis Banos 2019-02-12 21:46:49 +00:00
parent 660989939e
commit 99fb998e1d
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/dedup.py
View File

@ -318,7 +318,9 @@ class CdxServerDedupLoader(warcprox.BaseBatchPostfetchProcessor, DedupableMixin)
digest_key = warcprox.digest_str(recorded_url.payload_digest, digest_key = warcprox.digest_str(recorded_url.payload_digest,
self.options.base32) self.options.base32)
dedup_info = self.cdx_dedup.cached_lookup(digest_key, recorded_url.url) dedup_info = self.cdx_dedup.cached_lookup(digest_key, recorded_url.url)
self.logger.info(self.cdx_dedup.cached_lookup.cache_info()) cache_info = self.cdx_dedup.cached_lookup.cache_info()
if (cache_info.hits + cache_info.misses) % 1000 == 0:
self.logger.info(self.cdx_dedup.cached_lookup.cache_info())
if dedup_info: if dedup_info:
recorded_url.dedup_info = dedup_info recorded_url.dedup_info = dedup_info
except ValueError as exc: except ValueError as exc: