backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

default values for dedup_min_text_size et al

Browse Source 
because they may be missing in case warcprox is used as a library
This commit is contained in:
Noah Levitt 2018-05-15 11:22:10 -07:00
parent 15830fc5a2
commit af863c6dba
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev170', version='2.4b2.dev171',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

6
warcprox/dedup.py
View File

@ -39,9 +39,9 @@ urllib3.disable_warnings()
class DedupableMixin(object): class DedupableMixin(object):
def __init__(self, options=warcprox.Options()): def __init__(self, options=warcprox.Options()):
self.min_text_size = options.dedup_min_text_size self.min_text_size = options.dedup_min_text_size or 0
self.min_binary_size = options.dedup_min_binary_size self.min_binary_size = options.dedup_min_binary_size or 0
self.dedup_only_with_bucket = options.dedup_only_with_bucket self.dedup_only_with_bucket = options.dedup_only_with_bucket or False
def should_dedup(self, recorded_url): def should_dedup(self, recorded_url):
"""Check if we should try to run dedup on resource based on payload """Check if we should try to run dedup on resource based on payload