From 00f48d6566cf84790dbbd092be275dab2149501b Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 5 Jul 2016 18:45:17 -0500 Subject: [PATCH] less verbose logging about updating big captures table --- setup.py | 2 +- warcprox/bigtable.py | 57 ++++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index d785cbc..704c598 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ except: setuptools.setup( name='warcprox', - version='2.0.dev23', + version='2.0.dev24', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/bigtable.py b/warcprox/bigtable.py index 66b84f0..a1ac377 100644 --- a/warcprox/bigtable.py +++ b/warcprox/bigtable.py @@ -1,26 +1,26 @@ -# -# warcprox/bigtable.py - module for "big" RethinkDB table for deduplication; -# the table is "big" in the sense that it is designed to be usable as an index -# for playback software outside of warcprox, and contains information not -# needed merely for deduplication -# -# Copyright (C) 2015-2016 Internet Archive -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. -# +""" +warcprox/bigtable.py - module for "big" RethinkDB table for deduplication; +the table is "big" in the sense that it is designed to be usable as an index +for playback software outside of warcprox, and contains information not +needed merely for deduplication + +Copyright (C) 2015-2016 Internet Archive + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +USA. +""" from __future__ import absolute_import @@ -63,9 +63,14 @@ class RethinkCaptures: with self._batch_lock: if len(self._batch) > 0: result = self.r.table(self.table).insert(self._batch).run() - if result["inserted"] != len(self._batch) or sorted(result.values()) != [0,0,0,0,0,len(self._batch)]: - raise Exception("unexpected result %s saving batch of %s entries", result, len(self._batch)) - self.logger.info("saved %s entries to big capture table db", len(self._batch)) + if result["inserted"] != len(self._batch) or sorted( + result.values()) != [0,0,0,0,0,len(self._batch)]: + raise Exception( + "unexpected result %s saving batch of %s " + "entries", result, len(self._batch)) + self.logger.debug( + "saved %s entries to big capture table db", + len(self._batch)) self._batch = [] except BaseException as e: self.logger.error(