backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

be more robust and flexible updating the rethinkdb captures table

Browse Source
This commit is contained in:
Noah Levitt 2017-01-23 13:33:06 -08:00
parent af74959864
commit 884aa45066
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.0b3.dev42', version='2.0b3.dev43',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

16
warcprox/bigtable.py
View File

@ -66,15 +66,17 @@ class RethinkCaptures:
if len(self._batch) > 0: if len(self._batch) > 0:
result = self.r.table(self.table).insert( result = self.r.table(self.table).insert(
self._batch, conflict="replace").run() self._batch, conflict="replace").run()
if result["inserted"] + result["replaced"] != len(self._batch): if (result["inserted"] + result["replaced"]
+ result["unchanged"] != len(self._batch)):
raise Exception( raise Exception(
"unexpected result %s saving batch of %s " "unexpected result saving batch of %s: %s "
"entries", result, len(self._batch)) "entries" % (len(self._batch), result))
if result["replaced"] > 0: if result["replaced"] > 0 or result["unchanged"] > 0:
self.logger.warn( self.logger.warn(
"inserted %s entries and replaced %s entries " "inserted=%s replaced=%s unchanged=%s in big "
"in big captures table", result["inserted"], "captures table (normally replaced=0 and "
result["replaced"]) "unchanged=0)", result["inserted"],
result["replaced"], result["unchanged"])
else: else:
self.logger.debug( self.logger.debug(
"inserted %s entries to big captures table", "inserted %s entries to big captures table",