backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

be more robust and flexible updating the rethinkdb captures table

Browse Source
This commit is contained in:
Noah Levitt 2017-01-23 13:33:06 -08:00
parent af74959864
commit 884aa45066
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup(
name='warcprox',
version='2.0b3.dev42',
version='2.0b3.dev43',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',

16
warcprox/bigtable.py
View File

@ -66,15 +66,17 @@ class RethinkCaptures:
if len(self._batch) > 0:
result = self.r.table(self.table).insert(
self._batch, conflict="replace").run()
if result["inserted"] + result["replaced"] != len(self._batch):
if (result["inserted"] + result["replaced"]
+ result["unchanged"] != len(self._batch)):
raise Exception(
"unexpected result %s saving batch of %s "
"entries", result, len(self._batch))
if result["replaced"] > 0:
"unexpected result saving batch of %s: %s "
"entries" % (len(self._batch), result))
if result["replaced"] > 0 or result["unchanged"] > 0:
self.logger.warn(
"inserted %s entries and replaced %s entries "
"in big captures table", result["inserted"],
result["replaced"])
"inserted=%s replaced=%s unchanged=%s in big "
"captures table (normally replaced=0 and "
"unchanged=0)", result["inserted"],
result["replaced"], result["unchanged"])
else:
self.logger.debug(
"inserted %s entries to big captures table",