backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'WT-31' into qa

Browse Source
This commit is contained in:
Barbara Miller 2021-12-06 17:30:25 -08:00
commit 85bb6ff437
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
warcprox/__init__.py
View File

@ -1,7 +1,7 @@
""" """
warcprox/__init__.py - warcprox package main file, contains some utility code warcprox/__init__.py - warcprox package main file, contains some utility code
Copyright (C) 2013-2019 Internet Archive Copyright (C) 2013-2021 Internet Archive
This program is free software; you can redistribute it and/or This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License modify it under the terms of the GNU General Public License
@ -175,8 +175,8 @@ class BaseStandardPostfetchProcessor(BasePostfetchProcessor):
class BaseBatchPostfetchProcessor(BasePostfetchProcessor): class BaseBatchPostfetchProcessor(BasePostfetchProcessor):
MAX_BATCH_SIZE = 500 MAX_BATCH_SIZE = 500
MAX_BATCH_SEC = 10 MAX_BATCH_SEC = 30
MIN_BATCH_SEC = 2.0 MIN_BATCH_SEC = 10
def _get_process_put(self): def _get_process_put(self):
batch = [] batch = []

5
warcprox/dedup.py
View File

@ -384,6 +384,9 @@ class BatchTroughStorer(warcprox.BaseBatchPostfetchProcessor):
self.trough_dedup_db.batch_save, self.trough_dedup_db.batch_save,
buckets[bucket], bucket) buckets[bucket], bucket)
fs[future] = bucket fs[future] = bucket
logging.debug(
'storing dedup info for %s urls '
'in bucket %s', len(buckets[bucket]), bucket)
# wait for results # wait for results
try: try:
@ -434,6 +437,8 @@ class BatchTroughLoader(warcprox.BaseBatchPostfetchProcessor):
warcprox.digest_str( warcprox.digest_str(
recorded_url.payload_digest, self.options.base32) recorded_url.payload_digest, self.options.base32)
if recorded_url.payload_digest else 'n/a') if recorded_url.payload_digest else 'n/a')
self.logger.debug(
'hash_plus_urls: {}'.format(hash_plus_urls))
self.logger.debug( self.logger.debug(
'len(batch)=%s len(discards)=%s buckets=%s', 'len(batch)=%s len(discards)=%s buckets=%s',
len(batch), len(discards), len(batch), len(discards),