From a87615202686a761c27bb393865caedf5a5c4b9e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Sat, 1 Aug 2015 00:08:01 +0000 Subject: [PATCH] fix exception, make some tweaks --- warcprox/dedup.py | 2 +- warcprox/main.py | 2 +- warcprox/tests/test_warcprox.py | 2 +- warcprox/warcproxy.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/warcprox/dedup.py b/warcprox/dedup.py index 65962f9..2a99358 100644 --- a/warcprox/dedup.py +++ b/warcprox/dedup.py @@ -59,7 +59,7 @@ class DedupDb(object): return result def decorate_with_dedup_info(dedup_db, recorded_url, base32=False): - if recorded_url.response_recorder.payload_digest: + if recorded_url.response_recorder and recorded_url.response_recorder.payload_digest: key = warcprox.digest_str(recorded_url.response_recorder.payload_digest, base32) recorded_url.dedup_info = dedup_db.lookup(key) diff --git a/warcprox/main.py b/warcprox/main.py index 58d6a77..52553f9 100644 --- a/warcprox/main.py +++ b/warcprox/main.py @@ -58,7 +58,7 @@ def _build_arg_parser(prog=os.path.basename(sys.argv[0])): arg_parser.add_argument('-j', '--dedup-db-file', dest='dedup_db_file', default='./warcprox-dedup.db', help='persistent deduplication database file; empty string or /dev/null disables deduplication') arg_parser.add_argument('--stats-db-file', dest='stats_db_file', - default='./warcprox-stats.db', help='persistent statistics database file; empty string or /dev/null disables deduplication') + default='./warcprox-stats.db', help='persistent statistics database file; empty string or /dev/null disables statistics tracking') arg_parser.add_argument('-P', '--playback-port', dest='playback_port', default=None, help='port to listen on for instant playback') arg_parser.add_argument('--playback-index-db-file', dest='playback_index_db_file', diff --git a/warcprox/tests/test_warcprox.py b/warcprox/tests/test_warcprox.py index 0e6c3b9..3b05a56 100755 --- a/warcprox/tests/test_warcprox.py +++ b/warcprox/tests/test_warcprox.py @@ -409,7 +409,7 @@ def test_limits(http_daemon, archiving_proxies): response = requests.get(url, proxies=archiving_proxies, headers=headers, stream=True) assert response.status_code == 420 - assert response.reason == "Limit reached" + assert response.reason == "Reached limit" expected_response_meta = {'reached-limit': {'job1.total.urls': 10}, 'stats': {'job1': {'revisit': {'wire_bytes': 1215, 'urls': 9}, 'total': {'wire_bytes': 1350, 'urls': 10}, 'new': {'wire_bytes': 135, 'urls': 1}}}} assert json.loads(response.headers["warcprox-meta"]) == expected_response_meta assert response.headers["content-type"] == "text/plain;charset=utf-8" diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py index 6e5ccff..91891d1 100644 --- a/warcprox/warcproxy.py +++ b/warcprox/warcproxy.py @@ -165,9 +165,9 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler): bucket0, bucket1, bucket2 = key.rsplit(".", 2) value = self.server.stats_db.value(bucket0, bucket1, bucket2) if value and value >= limit: - self.logger.info('sending "420 Limit reached" %s=%s', key, limit) + self.logger.info('sending "420 Reached limit" %s=%s', key, limit) body = "request rejected by warcprox: reached limit {}={}\n".format(key, limit).encode("utf-8") - self.send_response(420, "Limit reached") + self.send_response(420, "Reached limit") self.send_header("Content-Type", "text/plain;charset=utf-8") self.send_header("Connection", "close") self.send_header("Content-Length", len(body))