From af863c6dba906d123627a6cc03f106d2baca8124 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 15 May 2018 11:22:10 -0700 Subject: [PATCH] default values for dedup_min_text_size et al because they may be missing in case warcprox is used as a library --- setup.py | 2 +- warcprox/dedup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 4579b12..a7fdb47 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev170', + version='2.4b2.dev171', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/dedup.py b/warcprox/dedup.py index f979d97..5d5039f 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -39,9 +39,9 @@ urllib3.disable_warnings() class DedupableMixin(object): def __init__(self, options=warcprox.Options()): - self.min_text_size = options.dedup_min_text_size - self.min_binary_size = options.dedup_min_binary_size - self.dedup_only_with_bucket = options.dedup_only_with_bucket + self.min_text_size = options.dedup_min_text_size or 0 + self.min_binary_size = options.dedup_min_binary_size or 0 + self.dedup_only_with_bucket = options.dedup_only_with_bucket or False def should_dedup(self, recorded_url): """Check if we should try to run dedup on resource based on payload