proxy: add 'extra_headers' to be added to proxy responses, customizable via proxy_options

defaults include no-cache and p3p policy (needed for IE default settings)
fix link generation for proxy_select page, better exception handling of ssl errors

pywb/framework/proxy.py

@@ -60,6 +60,9 @@ class ProxyRouter(object):
     CERT_DL_PEM = '/pywb-ca.pem'
     CERT_DL_P12 = '/pywb-ca.p12'
 
+    EXTRA_HEADERS = {'cache-control': 'no-cache',
+                     'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'}
+
     def __init__(self, routes, **kwargs):
         self.hostpaths = kwargs.get('hostpaths')
 
@@ -74,6 +77,11 @@ class ProxyRouter(object):
             self.magic_name = self.DEF_MAGIC_NAME
             proxy_options['magic_name'] = self.magic_name
 
+        self.extra_headers = proxy_options.get('extra_headers')
+        if not self.extra_headers:
+            self.extra_headers = self.EXTRA_HEADERS
+            proxy_options['extra_headers'] = self.extra_headers
+
         if proxy_options.get('cookie_resolver'):
             self.resolver = CookieResolver(routes, proxy_options)
         else:
@@ -198,7 +206,7 @@ class ProxyRouter(object):
         response = route.handler(wbrequest)
 
         if wbrequest.wb_url and wbrequest.wb_url.is_replay():
-            response.status_headers.replace_header('Cache-Control', 'no-cache')
+            response.status_headers.replace_headers(self.extra_headers)
 
         return response
 
@@ -252,19 +260,22 @@ class ProxyRouter(object):
                                        server_side=True,
                                        certfile=certfile,
                                        ciphers="ALL",
+                                       suppress_ragged_eofs=False,
+                                       #ssl_version=ssl.PROTOCOL_TLSv1)
                                        ssl_version=ssl.PROTOCOL_SSLv23)
+            env['pywb.proxy_ssl_sock'] = ssl_sock
+
+            buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
+
+            statusline = buffreader.readline().rstrip()
+
         except Exception as se:
             raise BadRequestException(se.message)
 
-        env['pywb.proxy_ssl_sock'] = ssl_sock
-
-        buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
-
-        statusline = buffreader.readline()
         statusparts = statusline.split(' ')
 
         if len(statusparts) < 3:
-            raise BadRequestException('Invalid Proxy Request')
+            raise BadRequestException('Invalid Proxy Request: ' + statusline)
 
         env['REQUEST_METHOD'] = statusparts[0]
         env['REL_REQUEST_URI'] = ('https://' +

pywb/framework/proxy_resolvers.py

@@ -132,6 +132,8 @@ class CookieResolver(BaseCollResolver):  # pragma: no cover
         self.cookie_name = config.get('cookie_name', '__pywb_coll')
         self.proxy_select_view = config.get('proxy_select_view')
 
+        self.extra_headers = config.get('extra_headers')
+
         if uwsgi_cache:
             self.cache = UwsgiCache()
         else:
@@ -222,14 +224,17 @@ class CookieResolver(BaseCollResolver):  # pragma: no cover
 
             coll, ts, sesh_id = self.get_coll(env)
 
-            route_temp = env['pywb.proxy_scheme'] + '://%s-set.'
-            route_temp += self.magic_name + '/' + path_url
+            #scheme = env['pywb.proxy_scheme'] + '://'
+            route_temp = '-set.' + self.magic_name + '/' + path_url
 
-            return (self.proxy_select_view.
-                    render_response(routes=self.routes,
-                                    route_temp=route_temp,
-                                    coll=coll,
-                                    url=path_url))
+            try:
+                return (self.proxy_select_view.
+                        render_response(routes=self.routes,
+                                        route_temp=route_temp,
+                                        coll=coll,
+                                        url=path_url))
+            except Exception as exc:
+                raise
 
         #else:
         #    msg = 'Invalid Magic Path: ' + url
@@ -301,6 +306,13 @@ class CookieResolver(BaseCollResolver):  # pragma: no cover
         return sesh_id
 
     def make_redir_response(self, url, headers=None):
+        if not headers:
+            headers = []
+
+        if self.extra_headers:
+            for name, value in self.extra_headers.iteritems():
+                headers.append((name, value))
+
         return WbResponse.redir_response(url, headers=headers)
 
     @staticmethod

pywb/ui/proxy_select.html

@@ -15,7 +15,7 @@ Current collection is: <b>{{ coll }}</b>
 <ul>
 {% for route in routes %}
 {% if route.path and route | is_wb_handler %}
-<li><a href="{{ route_temp | format(route.path) }}">{{ route.path }}</a></li>
+<li><a href="//{{ route.path }}{{ route_temp }}">{{ route.path }}</a></li>
 {% endif %}
 {% endfor %}
 </ul>

pywb/utils/statusandheaders.py

@@ -3,6 +3,7 @@ Representation and parsing of HTTP-style status + headers
 """
 
 import pprint
+from copy import copy
 
 
 #=================================================================
@@ -44,9 +45,26 @@ class StatusAndHeaders(object):
         self.headers.append((name, value))
         return None
 
+    def replace_headers(self, header_dict):
+        """
+        replace all headers in header_dict that already exist
+        add any remaining headers
+        """
+        header_dict = copy(header_dict)
+
+        for index in xrange(len(self.headers) - 1, -1, -1):
+            curr_name, curr_value = self.headers[index]
+            name_lower = curr_name.lower()
+            if name_lower in header_dict:
+                self.headers[index] = (curr_name, header_dict[name_lower])
+                del header_dict[name_lower]
+
+        for name, value in header_dict.iteritems():
+            self.headers.append((name, value))
+
     def remove_header(self, name):
         """
-        remove header (case-insensitive)
+        Remove header (case-insensitive)
         return True if header removed, False otherwise
         """
         name_lower = name.lower()