backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

enforce limits on WARCPROX_WRITE_RECORD requests

Browse Source 
should make test from previous commit pass
This commit is contained in:
Noah Levitt 2018-10-10 18:24:54 -07:00
parent 4c0dfb432e
commit 4f01772782
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup(
name='warcprox',
version='2.4b3.dev184',
version='2.4b3.dev185',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',

18
warcprox/warcproxy.py
View File

@ -170,6 +170,13 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
"request rejected by warcprox: slash and backslash are not "
"permitted in warc-prefix")
def _enforce_limits_and_blocks(self):
if 'Warcprox-Meta' in self.headers:
warcprox_meta = json.loads(self.headers['Warcprox-Meta'])
self._security_check(warcprox_meta)
self._enforce_limits(warcprox_meta)
self._enforce_blocks(warcprox_meta)
def _connect_to_remote_server(self):
'''
Wraps `MitmProxyHandler._connect_to_remote_server`, first enforcing
@ -178,11 +185,7 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
Otherwise calls `MitmProxyHandler._connect_to_remote_server`, which
initializes `self._remote_server_conn`.
'''
if 'Warcprox-Meta' in self.headers:
warcprox_meta = json.loads(self.headers['Warcprox-Meta'])
self._security_check(warcprox_meta)
self._enforce_limits(warcprox_meta)
self._enforce_blocks(warcprox_meta)
self._enforce_limits_and_blocks()
return warcprox.mitmproxy.MitmProxyHandler._connect_to_remote_server(self)
def _proxy_request(self):
@ -282,6 +285,7 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
'''
try:
self.url = self.path
self._enforce_limits_and_blocks()
if ('Content-Length' in self.headers and 'Content-Type' in self.headers
and (warc_type or 'WARC-Type' in self.headers)):
@ -331,6 +335,10 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
'request.'))
self.end_headers()
except warcprox.RequestBlockedByRule as e:
# limit enforcers have already sent the appropriate response
self.logger.info("%r: %r", self.requestline, e)
return
except:
self.logger.error("uncaught exception in do_WARCPROX_WRITE_RECORD", exc_info=True)
raise