From e23af32e94d5b33f6527cfc58c798385b2cab79f Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 15 May 2018 15:33:52 -0700 Subject: [PATCH] we want to save all captures to the big "captures" table, even if we don't want to dedup against them --- setup.py | 2 +- warcprox/bigtable.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a7fdb47..eefcdb4 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev171', + version='2.4b2.dev172', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/bigtable.py b/warcprox/bigtable.py index d8cd218..0d98270 100644 --- a/warcprox/bigtable.py +++ b/warcprox/bigtable.py @@ -253,6 +253,4 @@ class RethinkCapturesDedup(warcprox.dedup.DedupDb, DedupableMixin): self.captures_db.close() def notify(self, recorded_url, records): - if (records and records[0].type == b'response' - and self.should_dedup(recorded_url)): - self.captures_db.notify(recorded_url, records) + self.captures_db.notify(recorded_url, records)