backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

we want to save all captures to the big "captures"

Browse Source 
table, even if we don't want to dedup against them
This commit is contained in:
Noah Levitt 2018-05-15 15:33:52 -07:00
parent af863c6dba
commit e23af32e94
2 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev171', version='2.4b2.dev172',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
warcprox/bigtable.py
View File

@ -253,6 +253,4 @@ class RethinkCapturesDedup(warcprox.dedup.DedupDb, DedupableMixin):
self.captures_db.close() self.captures_db.close()
def notify(self, recorded_url, records): def notify(self, recorded_url, records):
if (records and records[0].type == b'response' self.captures_db.notify(recorded_url, records)
and self.should_dedup(recorded_url)):
self.captures_db.notify(recorded_url, records)