From 884aa45066d4c563db2215c8bd90a7ffd4dbf76d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 23 Jan 2017 13:33:06 -0800 Subject: [PATCH] be more robust and flexible updating the rethinkdb captures table --- setup.py | 2 +- warcprox/bigtable.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 67380c9..1338812 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ except: setuptools.setup( name='warcprox', - version='2.0b3.dev42', + version='2.0b3.dev43', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/bigtable.py b/warcprox/bigtable.py index aeca018..67638d4 100644 --- a/warcprox/bigtable.py +++ b/warcprox/bigtable.py @@ -66,15 +66,17 @@ class RethinkCaptures: if len(self._batch) > 0: result = self.r.table(self.table).insert( self._batch, conflict="replace").run() - if result["inserted"] + result["replaced"] != len(self._batch): + if (result["inserted"] + result["replaced"] + + result["unchanged"] != len(self._batch)): raise Exception( - "unexpected result %s saving batch of %s " - "entries", result, len(self._batch)) - if result["replaced"] > 0: + "unexpected result saving batch of %s: %s " + "entries" % (len(self._batch), result)) + if result["replaced"] > 0 or result["unchanged"] > 0: self.logger.warn( - "inserted %s entries and replaced %s entries " - "in big captures table", result["inserted"], - result["replaced"]) + "inserted=%s replaced=%s unchanged=%s in big " + "captures table (normally replaced=0 and " + "unchanged=0)", result["inserted"], + result["replaced"], result["unchanged"]) else: self.logger.debug( "inserted %s entries to big captures table",