From 56f0118374495f397de1f40b96b426a5c9789d44 Mon Sep 17 00:00:00 2001
From: Vangelis Banos <vangelis@archive.org>
Date: Tue, 31 Oct 2017 10:49:10 +0000
Subject: [PATCH] Replace timestamp parameter with more generic
 request/response syntax

Replace timestamp parameter with more generic extra_response_headers={}

When request has --header ``Warcprox-Meta: {\"accept\":[\"capture-metadata\"]}"``
Response has the following header:
``Warcprox-Meta: {"capture-metadata":{"timestamp":"2017-10-31T10:47:50Z"}}``

Update unit test
---
 tests/test_warcprox.py |  7 ++++---
 warcprox/mitmproxy.py  | 10 +++++-----
 warcprox/warcproxy.py  | 11 +++++------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/test_warcprox.py b/tests/test_warcprox.py
index 22d4597..1752b94 100755
--- a/tests/test_warcprox.py
+++ b/tests/test_warcprox.py
@@ -557,15 +557,16 @@ def test_limits(http_daemon, warcprox_, archiving_proxies):
 
 def test_return_capture_timestamp(http_daemon, warcprox_, archiving_proxies):
     url = 'http://localhost:{}/i/j'.format(http_daemon.server_port)
-    request_meta = {"return-capture-timestamp": 1}
+    request_meta = {"accept": ["capture-metadata"]}
     headers = {"Warcprox-Meta": json.dumps(request_meta)}
     response = requests.get(url, proxies=archiving_proxies, headers=headers, stream=True)
     assert response.status_code == 200
     assert response.headers['Warcprox-Meta']
     data = json.loads(response.headers['Warcprox-Meta'])
-    assert data['capture-timestamp']
+    assert data['capture-metadata']
     try:
-        dt = datetime.datetime.strptime(data['capture-timestamp'], '%Y-%m-%d %H:%M:%S')
+        dt = datetime.datetime.strptime(data['capture-metadata']['timestamp'],
+                                        '%Y-%m-%dT%H:%M:%SZ')
         assert dt
     except ValueError:
         pytest.fail('Invalid capture-timestamp format %s', data['capture-timestamp'])
diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py
index e60c07e..e2cc321 100644
--- a/warcprox/mitmproxy.py
+++ b/warcprox/mitmproxy.py
@@ -164,15 +164,15 @@ class ProxyingRecordingHTTPResponse(http_client.HTTPResponse):
                 self.fp, proxy_client, digest_algorithm, url=url)
         self.fp = self.recorder
 
-    def begin(self, timestamp=None):
+    def begin(self, extra_response_headers={}):
         http_client.HTTPResponse.begin(self)  # reads status line, headers
 
         status_and_headers = 'HTTP/1.1 {} {}\r\n'.format(
                 self.status, self.reason)
         self.msg['Via'] = via_header_value(
                 self.msg.get('Via'), '%0.1f' % (self.version / 10.0))
-        if timestamp:
-            rmeta = {"capture-timestamp": timestamp.strftime('%Y-%m-%d %H:%M:%S')}
+        if extra_response_headers:
+            rmeta = {"capture-metadata": extra_response_headers}
             self.msg['Warcprox-Meta'] = json.dumps(rmeta, separators=',:')
 
         for k,v in self.msg.items():
@@ -366,7 +366,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
             self.logger.error("exception proxying request", exc_info=True)
             raise
 
-    def _proxy_request(self, timestamp=None):
+    def _proxy_request(self, extra_response_headers={}):
         '''
         Sends the request to the remote server, then uses a ProxyingRecorder to
         read the response and send it to the proxy client, while recording the
@@ -415,7 +415,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
                     self._remote_server_sock, proxy_client=self.connection,
                     digest_algorithm=self.server.digest_algorithm,
                     url=self.url, method=self.command)
-            prox_rec_res.begin(timestamp=timestamp)
+            prox_rec_res.begin(extra_response_headers=extra_response_headers)
 
             buf = prox_rec_res.read(8192)
             while buf != b'':
diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py
index 48dc5cd..ec613ab 100644
--- a/warcprox/warcproxy.py
+++ b/warcprox/warcproxy.py
@@ -179,14 +179,13 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
 
         remote_ip = self._remote_server_sock.getpeername()[0]
         timestamp = datetime.datetime.utcnow()
-
-        if warcprox_meta and 'return-capture-timestamp' in warcprox_meta:
-            return_timestamp = timestamp
-        else:
-            return_timestamp = None
+        extra_response_headers = {}
+        if warcprox_meta and 'accept' in warcprox_meta and \
+                'capture-metadata' in warcprox_meta['accept']:
+            extra_response_headers['timestamp'] = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
 
         req, prox_rec_res = warcprox.mitmproxy.MitmProxyHandler._proxy_request(
-                self, timestamp=return_timestamp)
+                self, extra_response_headers=extra_response_headers)
 
         content_type = None
         try: