From 05daafa19eeb454691cfda25a61149ce22fb584e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 3 Mar 2022 18:46:20 -0800 Subject: [PATCH 1/2] increase MIN_BATCH_SEC, MAX_BATCH_SEC --- warcprox/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/warcprox/__init__.py b/warcprox/__init__.py index 9fe3a74..88357b8 100644 --- a/warcprox/__init__.py +++ b/warcprox/__init__.py @@ -175,8 +175,8 @@ class BaseStandardPostfetchProcessor(BasePostfetchProcessor): class BaseBatchPostfetchProcessor(BasePostfetchProcessor): MAX_BATCH_SIZE = 500 - MAX_BATCH_SEC = 30 - MIN_BATCH_SEC = 10 + MAX_BATCH_SEC = 60 + MIN_BATCH_SEC = 30 def _get_process_put(self): batch = [] From 8418fe10ba976d99f9a0edacbca52ff43dcb0716 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 24 Jun 2022 11:07:35 -0700 Subject: [PATCH 2/2] add explanatory comment --- warcprox/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/warcprox/__init__.py b/warcprox/__init__.py index 88357b8..33d7db3 100644 --- a/warcprox/__init__.py +++ b/warcprox/__init__.py @@ -177,6 +177,8 @@ class BaseBatchPostfetchProcessor(BasePostfetchProcessor): MAX_BATCH_SIZE = 500 MAX_BATCH_SEC = 60 MIN_BATCH_SEC = 30 + # these updated batch seconds values have resulted in fewer reported dedup + # errors and otherwise have worked well in qa def _get_process_put(self): batch = []