Merge branch 'develop' into https-proxy

pywb/rewrite/rewrite_live.py

@@ -172,10 +172,12 @@ class LiveRewriter(object):
                                   urlkey=urlkey,
                                   cdx=cdx))
 
+        if env:
+            env['pywb.cdx'] = cdx
+
         return result
 
     def get_rewritten(self, *args, **kwargs):
-
         result = self.fetch_request(*args, **kwargs)
 
         status_headers, gen, is_rewritten = result

pywb/static/wb.js

@@ -66,7 +66,10 @@ function init_banner() {
     
     text = "<span id='_wb_label'>" + text + "</span>";
 
-    var capture_str = (wbinfo ? wbinfo.capture_str : "");
+    var capture_str = "";
+    if (wbinfo && wbinfo.timestamp) {
+        capture_str = ts_to_date(wbinfo.timestamp, true);
+    }
 
     text += "<b id='_wb_capture_info'>" + capture_str + "</b>";
 
@@ -83,6 +86,27 @@ function init_banner() {
     document.body.insertBefore(banner, document.body.firstChild);
 }
 
+function ts_to_date(ts, is_gmt)
+{
+    if (ts.length < 14) {
+        return ts;
+    }
+    
+    var datestr = (ts.substring(0, 4) + "-" + 
+                  ts.substring(4, 6) + "-" +
+                  ts.substring(6, 8) + "T" +
+                  ts.substring(8, 10) + ":" +
+                  ts.substring(10, 12) + ":" +
+                  ts.substring(12, 14) + "-00:00");
+    
+    var date = new Date(datestr);
+    if (is_gmt) {
+        return date.toGMTString();
+    } else {
+        return date.toLocaleString();
+    }
+}
+
 function add_event(name, func, object) {
     if (object.addEventListener) {
         object.addEventListener(name, func);
@@ -107,49 +131,15 @@ function remove_event(name, func, object) {
     }
 }
 
-function notify_top(event) {    
-    if (window.self == window.top) {
-        return;
-    } 
-    
-    if (window.top.top != window.top) {
-        return;
-    }
-    
-    if (!window.WB_wombat_location) {
-        return;
-    }
-    
-    if (wbinfo.is_embed) {
-        return;
-    }
-    
-    if (event.target != window.document) {
-        return;
-    }
-
-    if (typeof(window.WB_wombat_location.href) != "string") {
-        return;
-    }
-
-    if (window.top.update_wb_url) {
-        window.top.update_wb_url(window.WB_wombat_location.href,
-                                 wbinfo.timestamp,
-                                 wbinfo.capture_str,
-                                 wbinfo.is_live);
-    }
-}
-
 var detect_on_init = function(event) {
-    init_banner();  
+    init_banner();
-    notify_top(event);    
+
     remove_event("readystatechange", detect_on_init, document);
 }
 
 add_event("readystatechange", detect_on_init, document);
 
 
-
 if (wbinfo.is_frame_mp && wbinfo.canon_url &&
    (window.self == window.top) && (window.self.top == window.top) &&
    window.location.href != wbinfo.canon_url) {
@@ -157,6 +147,7 @@ if (wbinfo.is_frame_mp && wbinfo.canon_url &&
     window.location.replace(wbinfo.canon_url);
 }
 
-return {'labels': labels};
+return {'labels': labels,
+        'ts_to_date': ts_to_date};
 
 })();

pywb/static/wb_frame.js

@@ -0,0 +1,129 @@
+var update_wb_url = push_state;
+
+var LIVE_COOKIE_REGEX = /pywb.timestamp=([\d]{1,14})/;
+
+
+function make_outer_url(url, ts)
+{
+    if (ts) {
+        return wbinfo.prefix + ts + "/" + url;
+    } else {
+        return wbinfo.prefix + url;
+    }
+}
+
+function make_inner_url(url, ts)
+{
+    if (ts) {
+        return wbinfo.prefix + ts + "mp_/" + url;
+    } else {
+        return wbinfo.prefix + "mp_/" + url;
+    }
+}
+
+function push_state(url, timestamp, capture_str, is_live) {
+    var state = {}
+    state.timestamp = timestamp;
+    state.outer_url = make_outer_url(url, state.timestamp);
+    state.inner_url = make_inner_url(url, state.timestamp);
+    state.url = url;
+    state.capture_str = capture_str;
+    state.is_live = is_live;
+    
+    window.history.replaceState(state, "", state.outer_url);
+
+    update_status(state.capture_str, is_live);
+}
+
+function pop_state(state) {
+    update_status(state.capture_str, state.is_live);
+    
+    window.frames[0].src = state.outer_url;
+}
+
+function extract_ts(url)
+{
+    var inx = url.indexOf("mp_");
+    if (inx < 0) {
+        return "";
+    }
+    url = url.substring(0, inx);
+    inx = url.lastIndexOf("/");
+    if (inx <= 0) {
+        return "";
+    }
+    return url.substring(inx + 1);
+}
+
+function extract_replay_url(url) {
+    var inx = url.indexOf("/http:");
+    if (inx < 0) {
+        inx = url.indexOf("/https:");
+        if (inx < 0) {
+            return "";
+        }
+    }
+    return url.substring(inx + 1);
+}
+
+function update_status(str, is_live) {
+    var capture_info = document.getElementById("_wb_capture_info");
+    if (capture_info) {
+        capture_info.innerHTML = str;
+    }
+
+    var label = document.getElementById("_wb_label");
+    if (label) {
+        if (is_live) {
+            label.innerHTML = _wb_js.labels.LIVE_MSG;
+        } else {
+            label.innerHTML = _wb_js.labels.REPLAY_MSG;
+        }
+    }
+}
+
+window.onpopstate = function(event) {
+    var curr_state = event.state;
+    
+    if (curr_state) {
+        pop_state(curr_state);
+    }
+}
+
+function extract_ts_cookie(value) {
+    var result = value.match(LIVE_COOKIE_REGEX);
+    if (result) {
+        return result[1];
+    } else {
+        return "";
+    }
+}
+
+function iframe_loaded(event) {
+    var iframe = window.frames[0];
+    var url;
+    var ts;
+    var capture_str;
+    var is_live = false;
+
+    if (iframe.WB_wombat_location) {
+        url = window.WB_wombat_location.href;
+    } else {
+        url = extract_replay_url(iframe.location.href);
+    }
+
+    if (iframe.wbinfo) {
+        ts = iframe.wbinfo.timestamp;
+        is_live = iframe.wbinfo.is_live;
+    } else {
+        ts = extract_ts_cookie(iframe.document.cookie);
+        if (ts) {
+            is_live = true;
+        } else {
+            ts = extract_ts(iframe.location.href);
+        }
+    }
+    capture_str = _wb_js.ts_to_date(ts, true);
+
+    update_wb_url(url, ts, capture_str, is_live);
+}

pywb/ui/frame_insert.html

@@ -3,114 +3,18 @@
 <!-- Start WB Insert -->
 <script>
   wbinfo = {}
-  wbinfo.capture_str = " ";
   wbinfo.is_embed = false;
   wbinfo.prefix = "{{ wbrequest.wb_prefix }}";
   wbinfo.capture_url = "{{ url }}";
   wbinfo.is_frame = true;
 </script>
 <script src='{{ wbrequest.host_prefix }}/{{ static_path }}/wb.js'> </script>
-<script>
+<script src='{{ wbrequest.host_prefix }}/{{ static_path }}/wb_frame.js'> </script>
-
-var update_wb_url = push_state;
-
-function make_outer_url(url, ts)
-{
-    if (ts) {
-        return wbinfo.prefix + ts + "/" + url;
-    } else {
-        return wbinfo.prefix + url;
-    }
-}
-
-function make_inner_url(url, ts)
-{
-    if (ts) {
-        return wbinfo.prefix + ts + "mp_/" + url;
-    } else {
-        return wbinfo.prefix + "mp_/" + url;
-    }
-}
-
-function push_state(url, timestamp, capture_str, is_live) {
-    var curr_href = null;
-    
-    if (window.frames[0].WB_wombat_location) {
-        curr_href = window.frames[0].WB_wombat_location.href;
-    }
-        
-    if (url != curr_href) {
-        update_status(capture_str, is_live);
-        return;
-    }
-    
-    if (!timestamp) {
-        timestamp = extract_ts(window.frames[0].location.href);
-    }
-    
-    var state = {}
-    state.timestamp = timestamp;
-    state.outer_url = make_outer_url(url, state.timestamp);
-    state.inner_url = make_inner_url(url, state.timestamp);
-    state.url = url;
-    state.capture_str = capture_str;
-    state.is_live = is_live;
-    
-    window.history.replaceState(state, "", state.outer_url);
-
-    update_status(state.capture_str, is_live);
-}
-
-function pop_state(state) {
-    update_status(state.capture_str, state.is_live);
-    
-    window.frames[0].src = state.outer_url;
-}
-
-function extract_ts(url)
-{
-    var inx = url.indexOf("mp_");
-    if (inx < 0) {
-        return "";
-    }
-    url = url.substring(0, inx);
-    inx = url.lastIndexOf("/");
-    if (inx <= 0) {
-        return "";
-    }
-    return url.substring(inx + 1);
-}
-
-function update_status(str, is_live) {
-    var capture_info = document.getElementById("_wb_capture_info");
-    if (capture_info) {
-        capture_info.innerHTML = str;
-    }
-
-    var label = document.getElementById("_wb_label");
-    if (label) {
-        if (is_live) {
-            label.innerHTML = _wb_js.labels.LIVE_MSG;
-        } else {
-            label.innerHTML = _wb_js.labels.REPLAY_MSG;
-        }
-    }
-}
-
-window.onpopstate = function(event) {
-    var curr_state = event.state;
-    
-    if (curr_state) {
-        pop_state(curr_state);
-    }
-}
-
-</script>
 <link rel='stylesheet' href='{{ wbrequest.host_prefix }}/{{ static_path }}/wb.css'/>
 <!-- End WB Insert -->
 <body style="margin: 0px; padding: 0px;">
 <div class="wb_iframe_div">
-<iframe src="{{ wbrequest.wb_prefix + embed_url }}" seamless="seamless" frameborder="0" scrolling="yes" class="wb_iframe"/>
+<iframe src="{{ wbrequest.wb_prefix + embed_url }}" onload="iframe_loaded(event);" seamless="seamless" frameborder="0" scrolling="yes" class="wb_iframe"/>
 </div>
 </body>
 </html>

pywb/ui/head_insert.html

@@ -14,7 +14,6 @@
   wbinfo = {}
   wbinfo.url = "{{ cdx.original }}";
   wbinfo.timestamp = "{{ cdx.timestamp }}";
-  wbinfo.capture_str = "{{ cdx.timestamp | format_ts }}";
   wbinfo.prefix = "{{ wbrequest.wb_prefix }}";
   wbinfo.is_embed = {{"true" if wbrequest.wb_url.is_embed else "false"}};
   wbinfo.is_frame_mp = {{"true" if wbrequest.wb_url.mod == 'mp_' else "false"}};

pywb/webapp/live_rewrite_handler.py

@@ -19,6 +19,9 @@ class LiveResourceException(WbException):
 
 #=================================================================
 class RewriteHandler(SearchPageWbUrlHandler):
+
+    LIVE_COOKIE = 'pywb.timestamp={0}; max-age=60';
+
     def __init__(self, config):
         super(RewriteHandler, self).__init__(config)
 
@@ -28,6 +31,8 @@ class RewriteHandler(SearchPageWbUrlHandler):
 
         self.head_insert_view = HeadInsertView.init_from_config(config)
 
+        self.live_cookie = config.get('live-cookie', self.LIVE_COOKIE)
+
     def handle_request(self, wbrequest):
         try:
             return self.render_content(wbrequest)
@@ -57,6 +62,14 @@ class RewriteHandler(SearchPageWbUrlHandler):
         return self._make_response(wbrequest, *result)
 
     def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
+        # if cookie set, pass recorded timestamp info via cookie
+        # so that client side may be able to access it
+        # used by framed mode to update frame banner
+        if self.live_cookie:
+            cdx = wbrequest.env['pywb.cdx']
+            value = self.live_cookie.format(cdx['timestamp'])
+            status_headers.headers.append(('Set-Cookie', value))
+
         return WbResponse(status_headers, gen)
 
     def __str__(self):

tests/test_integration.py

@@ -96,7 +96,7 @@ class TestWb:
         resp = self.testapp.get('/pywb/20140127171238mp_/http://www.iana.org/')
         self._assert_basic_html(resp)
 
-        assert 'Mon, Jan 27 2014 17:12:38' in resp.body
+        assert '"20140127171238"' in resp.body
         assert 'wb.js' in resp.body
         assert 'WB_wombat_init' in resp.body
         assert '/pywb/20140127171238mp_/http://www.iana.org/time-zones"' in resp.body
@@ -105,7 +105,7 @@ class TestWb:
         resp = self.testapp.get('/pywb-nonframe/20140127171238/http://www.iana.org/')
         self._assert_basic_html(resp)
 
-        assert 'Mon, Jan 27 2014 17:12:38' in resp.body
+        assert '"20140127171238"' in resp.body
         assert 'wb.js' in resp.body
         assert '/pywb-nonframe/20140127171238/http://www.iana.org/time-zones"' in resp.body
 
@@ -113,7 +113,7 @@ class TestWb:
         resp = self.testapp.get('/pywb-nosurt/20140103030321mp_/http://example.com?example=1')
         self._assert_basic_html(resp)
 
-        assert 'Fri, Jan 03 2014 03:03:21' in resp.body
+        assert '"20140103030321"' in resp.body
         assert 'wb.js' in resp.body
         assert '/pywb-nosurt/20140103030321mp_/http://www.iana.org/domains/example' in resp.body
 
@@ -121,7 +121,7 @@ class TestWb:
         resp = self.testapp.get('/pywb/20140603030341mp_/http://example.com?example=2')
         self._assert_basic_html(resp)
 
-        assert 'Tue, Jun 03 2014 03:03:41' in resp.body
+        assert '"20140603030341"' in resp.body
         assert 'wb.js' in resp.body
         assert '/pywb/20140603030341mp_/http://www.iana.org/domains/example' in resp.body
 
@@ -129,7 +129,7 @@ class TestWb:
         resp = self.testapp.get('/pywb/20130729195151mp_/http://www.example.com/')
         self._assert_basic_html(resp)
 
-        assert 'Mon, Jul 29 2013 19:51:51' in resp.body
+        assert '"20130729195151"' in resp.body
         assert 'wb.js' in resp.body
         assert '/pywb/20130729195151mp_/http://www.iana.org/domains/example"' in resp.body
 
@@ -215,7 +215,7 @@ class TestWb:
 
         #check resp
         self._assert_basic_html(resp)
-        assert 'Mon, Jan 27 2014 17:12:51' in resp.body
+        assert '"20140127171251"' in resp.body
         assert '/pywb/20140127171251mp_/http://www.iana.org/domains/example' in resp.body
 
     def test_redirect_relative_3(self):
@@ -337,7 +337,7 @@ class TestWb:
         resp = self.testapp.get('/x-ignore-this-x', extra_environ = dict(REQUEST_URI = 'http://www.iana.org/domains/idn-tables', SCRIPT_NAME = ''))
         self._assert_basic_html(resp)
 
-        assert 'Sun, Jan 26 2014 20:11:27' in resp.body
+        assert '"20140126201127"' in resp.body
         assert 'wb.js' in resp.body
 
     def test_proxy_replay_auth_filtered(self):
@@ -347,7 +347,7 @@ class TestWb:
 
         self._assert_basic_html(resp)
 
-        assert 'Sun, Jan 26 2014 20:06:24' in resp.body
+        assert '"20140126200624"' in resp.body
         assert 'wb.js' in resp.body
 
     def test_proxy_replay_auth(self):
@@ -357,7 +357,7 @@ class TestWb:
 
         self._assert_basic_html(resp)
 
-        assert 'Mon, Jan 27 2014 17:12:38' in resp.body
+        assert '"20140127171238"' in resp.body
         assert 'wb.js' in resp.body
 
     def test_proxy_replay_auth_no_coll(self):