From 4bb35567095dae3837c09738b382b63cfcbcfc52 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Tue, 10 May 2016 23:11:47 +0000
Subject: [PATCH] implement enforcement of Warcprox-Meta header block rules;
includes automated tests
---
setup.py | 2 +-
tests/test_warcprox.py | 159 +++++++++++++++++++++++++++++++++++------
warcprox/__init__.py | 53 ++++++++------
warcprox/main.py | 5 +-
warcprox/mitmproxy.py | 32 +++++----
warcprox/warcproxy.py | 146 +++++++++++++++++++++++++++++++++++--
6 files changed, 331 insertions(+), 66 deletions(-)
diff --git a/setup.py b/setup.py
index e543c33..6584d18 100755
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,7 @@ except:
deps.append('futures')
setuptools.setup(name='warcprox',
- version='2.0.dev8',
+ version='2.0.dev9',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',
diff --git a/tests/test_warcprox.py b/tests/test_warcprox.py
index a17d2ea..45933b5 100755
--- a/tests/test_warcprox.py
+++ b/tests/test_warcprox.py
@@ -1,24 +1,24 @@
#!/usr/bin/env python
-#
-# tests/test_warcprox.py - automated tests for warcprox
-#
-# Copyright (C) 2013-2016 Internet Archive
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-#
+'''
+tests/test_warcprox.py - automated tests for warcprox
+
+Copyright (C) 2013-2016 Internet Archive
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+USA.
+'''
import pytest
import threading
@@ -58,7 +58,8 @@ import certauth.certauth
import warcprox
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
- format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
+ format='%(asctime)s %(process)d %(levelname)s %(threadName)s '
+ '%(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
logging.getLogger("requests.packages.urllib3").setLevel(logging.WARN)
warnings.simplefilter("ignore", category=requests.packages.urllib3.exceptions.InsecureRequestWarning)
warnings.simplefilter("ignore", category=requests.packages.urllib3.exceptions.InsecurePlatformWarning)
@@ -137,8 +138,8 @@ def cert(request):
@pytest.fixture(scope="module")
def http_daemon(request):
- http_daemon = http_server.HTTPServer(('localhost', 0),
- RequestHandlerClass=_TestHttpRequestHandler)
+ http_daemon = http_server.HTTPServer(
+ ('localhost', 0), RequestHandlerClass=_TestHttpRequestHandler)
logging.info('starting http://{}:{}'.format(http_daemon.server_address[0], http_daemon.server_address[1]))
http_daemon_thread = threading.Thread(name='HttpDaemonThread',
target=http_daemon.serve_forever)
@@ -725,6 +726,118 @@ def test_dedup_buckets(https_daemon, http_daemon, warcprox_, archiving_proxies,
finally:
fh.close()
+def test_block_rules(http_daemon, https_daemon, warcprox_, archiving_proxies):
+ rules = [
+ {
+ "host": "localhost",
+ "url_match": "STRING_MATCH",
+ "value": "bar",
+ },
+ {
+ "url_match": "SURT_MATCH",
+ "value": "http://(localhost:%s,)/fuh/" % (http_daemon.server_port),
+ },
+ {
+ "url_match": "SURT_MATCH",
+ # this rule won't match because of http scheme, https port
+ "value": "http://(localhost:%s,)/fuh/" % (https_daemon.server_port),
+ },
+ {
+ "host": "badhost.com",
+ },
+ ]
+ request_meta = {"blocks":rules}
+ headers = {"Warcprox-Meta":json.dumps(request_meta)}
+
+ # blocked by STRING_MATCH rule
+ url = 'http://localhost:{}/bar'.format(http_daemon.server_port)
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[0]}
+
+ # not blocked
+ url = 'http://localhost:{}/m/n'.format(http_daemon.server_port)
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 200
+
+ # blocked by SURT_MATCH
+ url = 'http://localhost:{}/fuh/guh'.format(http_daemon.server_port)
+ # logging.info("%s => %s", repr(url), repr(warcprox.warcproxy.Url(url).surt))
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[1]}
+
+ # not blocked (no trailing slash)
+ url = 'http://localhost:{}/fuh'.format(http_daemon.server_port)
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ # 404 because server set up at the top of this file doesn't handle this url
+ assert response.status_code == 404
+
+ # not blocked because surt scheme does not match (differs from heritrix
+ # behavior where https urls are coerced to http surt form)
+ url = 'https://localhost:{}/fuh/guh'.format(https_daemon.server_port)
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True,
+ verify=False)
+ assert response.status_code == 200
+
+ # blocked by blanket host block
+ url = 'http://badhost.com/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+ # blocked by blanket host block
+ url = 'https://badhost.com/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True,
+ verify=False)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+ # blocked by blanket host block
+ url = 'http://badhost.com:1234/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+ # blocked by blanket host block
+ url = 'http://foo.bar.badhost.com/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+ # host block also applies to subdomains
+ url = 'https://foo.bar.badhost.com/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True,
+ verify=False)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+ # blocked by blanket host block
+ url = 'http://foo.bar.badhost.com:1234/'
+ response = requests.get(
+ url, proxies=archiving_proxies, headers=headers, stream=True)
+ assert response.status_code == 403
+ assert response.content.startswith(b"request rejected by warcprox: blocked by rule found in Warcprox-Meta header:")
+ assert json.loads(response.headers['warcprox-meta']) == {"blocked-by-rule":rules[3]}
+
+
# XXX this test relies on a tor proxy running at localhost:9050 with a working
# connection to the internet, and relies on a third party site (facebook) being
# up and behaving a certain way
diff --git a/warcprox/__init__.py b/warcprox/__init__.py
index 394d8b8..1eeb9a4 100644
--- a/warcprox/__init__.py
+++ b/warcprox/__init__.py
@@ -1,23 +1,23 @@
-#
-# warcprox/__init__.py - warcprox package main file, contains some utility code
-#
-# Copyright (C) 2013-2016 Internet Archive
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-#
+"""
+warcprox/__init__.py - warcprox package main file, contains some utility code
+
+Copyright (C) 2013-2016 Internet Archive
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+USA.
+"""
from argparse import Namespace as _Namespace
from pkg_resources import get_distribution as _get_distribution
@@ -47,6 +47,19 @@ def gettid():
except:
return "n/a"
+class RequestBlockedByRule(Exception):
+ """
+ An exception raised when a request should be blocked to respect a
+ Warcprox-Meta rule.
+ """
+ def __init__(self, msg):
+ self.msg = msg
+ def __str__(self):
+ return "%s: %s" % (self.__class__.__name__, self.msg)
+
+# logging level more fine-grained than logging.DEBUG==10
+TRACE = 5
+
import warcprox.controller as controller
import warcprox.playback as playback
import warcprox.dedup as dedup
diff --git a/warcprox/main.py b/warcprox/main.py
index d8529b5..00d2d85 100644
--- a/warcprox/main.py
+++ b/warcprox/main.py
@@ -109,6 +109,7 @@ def _build_arg_parser(prog=os.path.basename(sys.argv[0])):
arg_parser.add_argument('--version', action='version',
version="warcprox {}".format(warcprox.__version__))
arg_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true')
+ arg_parser.add_argument('--trace', dest='trace', action='store_true')
arg_parser.add_argument('-q', '--quiet', dest='quiet', action='store_true')
return arg_parser
@@ -232,7 +233,9 @@ def main(argv=sys.argv):
'''
args = parse_args(argv)
- if args.verbose:
+ if args.trace:
+ loglevel = warcprox.TRACE
+ elif args.verbose:
loglevel = logging.DEBUG
elif args.quiet:
loglevel = logging.WARNING
diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py
index 333bad7..3950e4e 100644
--- a/warcprox/mitmproxy.py
+++ b/warcprox/mitmproxy.py
@@ -241,6 +241,8 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self.hostname)
raise
+ return self._remote_server_sock
+
def _transition_to_ssl(self):
self.request = self.connection = ssl.wrap_socket(self.connection,
server_side=True, certfile=self.server.ca.cert_for_host(self.hostname))
@@ -262,9 +264,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
'''
self.is_connect = True
try:
- # Connect to destination first
self._determine_host_port()
- self._connect_to_remote_server()
# If successful, let's do this!
self.send_response(200, 'Connection established')
@@ -305,19 +305,23 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
return result
def do_COMMAND(self):
- if not self.is_connect:
- try:
- # Connect to destination
- self._determine_host_port()
- self._connect_to_remote_server()
- assert self.url
- except Exception as e:
- self.logger.error("problem processing request {}: {}".format(repr(self.requestline), e))
- self.send_error(500, str(e))
- return
- else:
- # if self.is_connect we already connected in do_CONNECT
+ if self.is_connect:
self.url = self._construct_tunneled_url()
+ else:
+ self._determine_host_port()
+ assert self.url
+
+ try:
+ # Connect to destination
+ self._connect_to_remote_server()
+ except warcprox.RequestBlockedByRule as e:
+ # limit enforcers have already sent the appropriate response
+ self.logger.info("%s: %s", repr(self.requestline), e)
+ return
+ except Exception as e:
+ self.logger.error("problem processing request {}: {}".format(repr(self.requestline), e))
+ self.send_error(500, str(e))
+ return
try:
self._proxy_request()
diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py
index 0549179..d342774 100644
--- a/warcprox/warcproxy.py
+++ b/warcprox/warcproxy.py
@@ -45,18 +45,135 @@ import warcprox
import datetime
import concurrent.futures
import resource
+import ipaddress
+import surt
+
+class Url:
+ def __init__(self, url):
+ self.url = url
+ self._surt = None
+ self._host = None
+
+ @property
+ def surt(self):
+ if not self._surt:
+ hurl = surt.handyurl.parse(self.url)
+ surt.GoogleURLCanonicalizer.canonicalize(hurl)
+ hurl.query = None
+ hurl.hash = None
+ self._surt = hurl.getURLString(surt=True, trailing_comma=True)
+ return self._surt
+
+ @property
+ def host(self):
+ if not self._host:
+ self._host = surt.handyurl.parse(self.url).host
+ return self._host
+
+ def matches_ip_or_domain(self, ip_or_domain):
+ """Returns true if
+ - ip_or_domain is an ip address and self.host is the same ip address
+ - ip_or_domain is a domain and self.host is the same domain
+ - ip_or_domain is a domain and self.host is a subdomain of it
+ """
+ if ip_or_domain == self.host:
+ return True
+
+ # if either ip_or_domain or self.host are ip addresses, and they're not
+ # identical (previous check), not a match
+ try:
+ ipaddress.ip_address(ip_or_domain)
+ return False
+ except:
+ pass
+ try:
+ ipaddress.ip_address(self.host)
+ return False
+ except:
+ pass
+
+ # if we get here, we're looking at two hostnames
+ # XXX do we need to handle case of one punycoded idn, other not?
+ domain_parts = ip_or_domain.split(".")
+ host_parts = self.host.split(".")
+
+ return host_parts[-len(domain_parts):] == domain_parts
class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
# self.server is WarcProxy
logger = logging.getLogger("warcprox.warcprox.WarcProxyHandler")
+ # XXX nearly identical to brozzler.site.Site._scope_rule_applies() but
+ # there's no obvious common dependency where this code should go... TBD
+ def _scope_rule_applies(self, rule):
+ u = Url(self.url)
+
+ if "host" in rule and not u.matches_ip_or_domain(rule["host"]):
+ return False
+ if "url_match" in rule:
+ if rule["url_match"] == "STRING_MATCH":
+ return u.url.find(rule["value"]) >= 0
+ elif rule["url_match"] == "REGEX_MATCH":
+ try:
+ return re.fullmatch(rule["value"], u.url)
+ except Exception as e:
+ self.logger.warn(
+ "caught exception matching against regex %s: %s",
+ rule["value"], e)
+ return False
+ elif rule["url_match"] == "SURT_MATCH":
+ return u.surt.startswith(rule["value"])
+ else:
+ self.logger.warn("invalid rule.url_match=%s", rule.url_match)
+ return False
+ else:
+ if "host" in rule:
+ # we already know that it matches from earlier check
+ return True
+ else:
+ self.logger.warn("unable to make sense of scope rule %s", rule)
+ return False
+
+ def _enforce_blocks(self, warcprox_meta):
+ """
+ Sends a 403 response and raises warcprox.RequestBlockedByRule if the
+ url is blocked by a rule in warcprox_meta.
+ """
+ if warcprox_meta and "blocks" in warcprox_meta:
+ for rule in warcprox_meta["blocks"]:
+ if self._scope_rule_applies(rule):
+ body = ("request rejected by warcprox: blocked by "
+ "rule found in Warcprox-Meta header: %s"
+ % rule).encode("utf-8")
+ self.send_response(403, "Forbidden")
+ self.send_header("Content-Type", "text/plain;charset=utf-8")
+ self.send_header("Connection", "close")
+ self.send_header("Content-Length", len(body))
+ response_meta = {"blocked-by-rule":rule}
+ self.send_header(
+ "Warcprox-Meta",
+ json.dumps(response_meta, separators=(",",":")))
+ self.end_headers()
+ if self.command != "HEAD":
+ self.wfile.write(body)
+ self.connection.close()
+ raise warcprox.RequestBlockedByRule(
+ "%s 403 %s %s -- blocked by rule in Warcprox-Meta "
+ "request header %s" % (
+ self.client_address[0], self.command,
+ self.url, rule))
+
def _enforce_limits(self, warcprox_meta):
+ """
+ Sends a 420 response and raises warcprox.RequestBlockedByRule if a
+ limit specified in warcprox_meta is reached.
+ """
if warcprox_meta and "limits" in warcprox_meta:
for item in warcprox_meta["limits"].items():
key, limit = item
bucket0, bucket1, bucket2 = key.rsplit(".", 2)
value = self.server.stats_db.value(bucket0, bucket1, bucket2)
- self.logger.debug("warcprox_meta['limits']=%s stats['%s']=%s recorded_url_q.qsize()=%s",
+ self.logger.debug("warcprox_meta['limits']=%s stats['%s']=%s recorded_url_q.qsize()=%s",
warcprox_meta['limits'], key, value, self.server.recorded_url_q.qsize())
if value and value >= limit:
body = "request rejected by warcprox: reached limit {}={}\n".format(key, limit).encode("utf-8")
@@ -70,20 +187,35 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
if self.command != "HEAD":
self.wfile.write(body)
self.connection.close()
- self.logger.info("%s 420 %s %s -- reached limit %s=%s", self.client_address[0], self.command, self.url, key, limit)
- return True
- return False
+ raise warcprox.RequestBlockedByRule(
+ "%s 420 %s %s -- reached limit %s=%s" % (
+ self.client_address[0], self.command,
+ self.url, key, limit))
+
+ def _connect_to_remote_server(self):
+ '''
+ Wraps MitmProxyHandler._connect_to_remote_server, first enforcing
+ limits and block rules in the Warcprox-Meta request header, if any.
+ Raises warcprox.RequestBlockedByRule if a rule has been enforced.
+ Otherwise calls MitmProxyHandler._connect_to_remote_server, which
+ initializes self._remote_server_sock.
+ '''
+ if 'Warcprox-Meta' in self.headers:
+ warcprox_meta = json.loads(self.headers['Warcprox-Meta'])
+ self._enforce_limits(warcprox_meta)
+ self._enforce_blocks(warcprox_meta)
+ return warcprox.mitmproxy.MitmProxyHandler._connect_to_remote_server(self)
def _proxy_request(self):
warcprox_meta = None
raw_warcprox_meta = self.headers.get('Warcprox-Meta')
+ self.logger.log(
+ warcprox.TRACE, 'request for %s Warcprox-Meta header: %s',
+ self.url, repr(raw_warcprox_meta))
if raw_warcprox_meta:
warcprox_meta = json.loads(raw_warcprox_meta)
del self.headers['Warcprox-Meta']
- if self._enforce_limits(warcprox_meta):
- return
-
remote_ip = self._remote_server_sock.getpeername()[0]
timestamp = datetime.datetime.utcnow()