backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Replace timestamp parameter with more generic request/response syntax

Browse Source 
Replace timestamp parameter with more generic extra_response_headers={}

When request has --header ``Warcprox-Meta: {\"accept\":[\"capture-metadata\"]}"``
Response has the following header:
``Warcprox-Meta: {"capture-metadata":{"timestamp":"2017-10-31T10:47:50Z"}}``

Update unit test
This commit is contained in:
Vangelis Banos 2017-10-31 10:49:10 +00:00
parent 3d9a22b6c7
commit 56f0118374
3 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/test_warcprox.py
View File

@ -557,15 +557,16 @@ def test_limits(http_daemon, warcprox_, archiving_proxies):
def test_return_capture_timestamp(http_daemon, warcprox_, archiving_proxies): def test_return_capture_timestamp(http_daemon, warcprox_, archiving_proxies):
url = 'http://localhost:{}/i/j'.format(http_daemon.server_port) url = 'http://localhost:{}/i/j'.format(http_daemon.server_port)
request_meta = {"return-capture-timestamp": 1} request_meta = {"accept": ["capture-metadata"]}
headers = {"Warcprox-Meta": json.dumps(request_meta)} headers = {"Warcprox-Meta": json.dumps(request_meta)}
response = requests.get(url, proxies=archiving_proxies, headers=headers, stream=True) response = requests.get(url, proxies=archiving_proxies, headers=headers, stream=True)
assert response.status_code == 200 assert response.status_code == 200
assert response.headers['Warcprox-Meta'] assert response.headers['Warcprox-Meta']
data = json.loads(response.headers['Warcprox-Meta']) data = json.loads(response.headers['Warcprox-Meta'])
assert data['capture-timestamp'] assert data['capture-metadata']
try: try:
dt = datetime.datetime.strptime(data['capture-timestamp'], '%Y-%m-%d %H:%M:%S') dt = datetime.datetime.strptime(data['capture-metadata']['timestamp'],
'%Y-%m-%dT%H:%M:%SZ')
assert dt assert dt
except ValueError: except ValueError:
pytest.fail('Invalid capture-timestamp format %s', data['capture-timestamp']) pytest.fail('Invalid capture-timestamp format %s', data['capture-timestamp'])

10
warcprox/mitmproxy.py
View File

@ -164,15 +164,15 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
self.fp, proxy_client, digest_algorithm, url=url) self.fp, proxy_client, digest_algorithm, url=url)
self.fp = self.recorder self.fp = self.recorder
def begin(self, timestamp=None): def begin(self, extra_response_headers={}):
http_client.HTTPResponse.begin(self) # reads status line, headers http_client.HTTPResponse.begin(self) # reads status line, headers
status_and_headers = 'HTTP/1.1 {} {}\r\n'.format( status_and_headers = 'HTTP/1.1 {} {}\r\n'.format(
self.status, self.reason) self.status, self.reason)
self.msg['Via'] = via_header_value( self.msg['Via'] = via_header_value(
self.msg.get('Via'), '%0.1f' % (self.version / 10.0)) self.msg.get('Via'), '%0.1f' % (self.version / 10.0))
if timestamp: if extra_response_headers:
rmeta = {"capture-timestamp": timestamp.strftime('%Y-%m-%d %H:%M:%S')} rmeta = {"capture-metadata": extra_response_headers}
self.msg['Warcprox-Meta'] = json.dumps(rmeta, separators=',:') self.msg['Warcprox-Meta'] = json.dumps(rmeta, separators=',:')
for k,v in self.msg.items(): for k,v in self.msg.items():
@ -366,7 +366,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self.logger.error("exception proxying request", exc_info=True) self.logger.error("exception proxying request", exc_info=True)
raise raise
def _proxy_request(self, timestamp=None): def _proxy_request(self, extra_response_headers={}):
''' '''
Sends the request to the remote server, then uses a ProxyingRecorder to Sends the request to the remote server, then uses a ProxyingRecorder to
read the response and send it to the proxy client, while recording the read the response and send it to the proxy client, while recording the
@ -415,7 +415,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self._remote_server_sock, proxy_client=self.connection, self._remote_server_sock, proxy_client=self.connection,
digest_algorithm=self.server.digest_algorithm, digest_algorithm=self.server.digest_algorithm,
url=self.url, method=self.command) url=self.url, method=self.command)
prox_rec_res.begin(timestamp=timestamp) prox_rec_res.begin(extra_response_headers=extra_response_headers)
buf = prox_rec_res.read(8192) buf = prox_rec_res.read(8192)
while buf != b'': while buf != b'':

11
warcprox/warcproxy.py
View File

@ -179,14 +179,13 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
remote_ip = self._remote_server_sock.getpeername()[0] remote_ip = self._remote_server_sock.getpeername()[0]
timestamp = datetime.datetime.utcnow() timestamp = datetime.datetime.utcnow()
extra_response_headers = {}
if warcprox_meta and 'return-capture-timestamp' in warcprox_meta: if warcprox_meta and 'accept' in warcprox_meta and \
return_timestamp = timestamp 'capture-metadata' in warcprox_meta['accept']:
else: extra_response_headers['timestamp'] = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
return_timestamp = None
req, prox_rec_res = warcprox.mitmproxy.MitmProxyHandler._proxy_request( req, prox_rec_res = warcprox.mitmproxy.MitmProxyHandler._proxy_request(
self, timestamp=return_timestamp) self, extra_response_headers=extra_response_headers)
content_type = None content_type = None
try: try: