Merge changes from 2.6.3 into 2.7.0b0

.github/workflows/release.yaml

@@ -0,0 +1,42 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Build pywb Docker image for release and push to Dockerhub
+    runs-on: ubuntu-latest
+    steps:
+      - 
+        name: Check out the repo
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: webrecorder/pywb
+          tags: |
+            type=match,pattern=v-(.*),group=1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+

CHANGES.rst

@@ -1,3 +1,19 @@
+pywb 2.6.3 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* Fix false-positive rewriting of ``location`` through additional check if local var is used, fixes `#684 <https://github.com/webrecorder/pywb/pull/684>`_
+
+* Fix missing localization of placeholder, fixes `#685 <https://github.com/webrecorder/pywb/pull/685>`_
+
+* Fix regression caused by 2.6.2, ensure pywb.app_prefix, pywb.host_prefix and pywb.static_prefix paths set correctly for all pages `#688 <https://github.com/webrecorder/pywb/pull/688>`_, fixes `#686 <https://github.com/webrecorder/pywb/pull/686>`_
+
+* Documentation: Fixes to ``cdx-indexer`` helped (from @ldko) `#683 <https://github.com/webrecorder/pywb/pull/683>`_
+
+* Update wombat.js to 3.3.6
+
+* Add automatic Docker push on new GitHub release
+
+
 pywb 2.6.2 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 

pywb/apps/frontendapp.py

@@ -646,8 +646,8 @@ class FrontEndApp(object):
         urls = self.url_map.bind_to_environ(environ)
         try:
             endpoint, args = urls.match()
-            # store original script_name (original prefix) before modifications are made
+
-            environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '')
+            self.rewriterapp.prepare_env(environ)
 
             # store original script_name (original prefix) before modifications are made
             environ['ORIG_SCRIPT_NAME'] = environ.get('SCRIPT_NAME')

pywb/apps/rewriterapp.py

@@ -303,15 +303,24 @@ class RewriterApp(object):
 
         return resp
 
-    def render_content(self, wb_url, kwargs, environ):
+    def prepare_env(self, environ):
-        wb_url = wb_url.replace('#', '%23')
+        """ setup environ path prefixes and scheme """
-        wb_url = WbUrl(wb_url)
+        if 'pywb.host_prefix' in environ:
+            return
 
         proto = environ.get('HTTP_X_FORWARDED_PROTO', self.force_scheme)
 
         if proto:
             environ['wsgi.url_scheme'] = proto
 
+        environ['pywb.host_prefix'] = self.get_host_prefix(environ)
+        environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '')
+        environ['pywb.static_prefix'] = environ['pywb.host_prefix'] + environ['pywb.app_prefix'] + '/' + self.static_prefix
+
+    def render_content(self, wb_url, kwargs, environ):
+        wb_url = wb_url.replace('#', '%23')
+        wb_url = WbUrl(wb_url)
+
         history_page = environ.pop('HTTP_X_WOMBAT_HISTORY_PAGE', '')
         if history_page:
             wb_url.url = history_page
@@ -321,13 +330,13 @@ class RewriterApp(object):
 
         is_timegate = self._check_accept_dt(wb_url, environ)
 
-        host_prefix = self.get_host_prefix(environ)
+        self.prepare_env(environ)
+
+        host_prefix = environ['pywb.host_prefix']
         rel_prefix = self.get_rel_prefix(environ)
         full_prefix = host_prefix + rel_prefix
-        environ['pywb.host_prefix'] = host_prefix
+
-        pywb_static_prefix = host_prefix + environ.get('pywb.app_prefix', '') + '/' + self.static_prefix
+        pywb_static_prefix = environ['pywb.static_prefix'] + '/'
-        environ['pywb.static_prefix'] = pywb_static_prefix
-        pywb_static_prefix += '/'
         is_proxy = ('wsgiprox.proxy_host' in environ)
 
         # if OPTIONS in proxy mode, just generate the proxy responss

pywb/rewrite/regex_rewriters.py

@@ -82,7 +82,7 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj;
         # By using a function the expression injected is an call expression that plays nice in those cases
         this_rw = '_____WB$wombat$check$this$function_____(this)'
 
-        check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = '
+        check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = '
 
         self.local_objs = [
             'window',
@@ -96,6 +96,7 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj;
         ]
 
         local_declares = '\n'.join([local_var_line.format(obj, local_init_func_name) for obj in self.local_objs])
+        local_declares += "\nlet arguments;"
 
         prop_str = '|'.join(self.local_objs)
 

pywb/rewrite/test/test_html_rewriter.py

@@ -394,7 +394,7 @@ r"""
 
 # parse attr with js proxy, rewrite location assignment
 >>> parse('<html><a href="javascript:location=\'foo.html\'"></a></html>', js_proxy=True)
-<html><a href="javascript:{ location=((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = 'foo.html' }"></a></html>
+<html><a href="javascript:{ location=((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = 'foo.html' }"></a></html>
 
 # parse attr with js proxy, assigning to location.href, no location assignment rewrite needed
 >>> parse('<html><a href="javascript:location.href=\'foo.html\'"></a></html>', js_proxy=True)

pywb/rewrite/test/test_regex_rewriters.py

@@ -131,13 +131,13 @@ r"""
 #=================================================================
 
 >>> _test_js_obj_proxy('var foo = this;   location = bar')
-'var foo = _____WB$wombat$check$this$function_____(this);   location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar'
+'var foo = _____WB$wombat$check$this$function_____(this);   location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = bar'
 
 >>> _test_js_obj_proxy('var that =    this\n   location = bar')
-'var that =    _____WB$wombat$check$this$function_____(this)\n   location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar'
+'var that =    _____WB$wombat$check$this$function_____(this)\n   location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = bar'
 
 >>> _test_js_obj_proxy('location = "xyz"')
-'location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = "xyz"'
+'location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = "xyz"'
 
 >>> _test_js_obj_proxy('var foo = this.location')
 'var foo = _____WB$wombat$check$this$function_____(this).location'
@@ -213,7 +213,7 @@ r"""
 'this. alocation = http://example.com/'
 
 >>> _test_js_obj_proxy(r'this. location = http://example.com/')
-'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = http://example.com/'
+'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = http://example.com/'
 
 >>> _test_js_obj_proxy('eval(a)')
 'WB_wombat_runEval(function _____evalIsEvil(_______eval_arg$$) { return eval(_______eval_arg$$); }.bind(this)).eval(a)'

pywb/templates/search.html

@@ -26,7 +26,7 @@ window.wb_prefix = "{{ wb_prefix }}";
                     {% trans %}Search the {{ coll_title }} collection by url:{% endtrans %}
                 </label>
                 <input aria-label="url" aria-required="true" class="form-control form-control-lg" id="search-url"
-                       name="search" placeholder="Enter a URL to search for"
+                       name="search" placeholder="{{ _('Enter a URL to search for') }}"
                        title="{{ _('Enter a URL to search for') }}" type="search" required/>
                 <div class="invalid-feedback">
                     {% trans %}'Please enter a URL{% endtrans %}

tests/test_integration.py

@@ -12,12 +12,25 @@ class TestWbIntegration(BaseConfigTest):
     def test_home(self):
         resp = self.testapp.get('/')
         self._assert_basic_html(resp)
+        assert '<link rel="stylesheet" href="http://localhost:80/static/css/base.css"' in resp.text
+        assert '/pywb' in resp.text
+
+    def test_home_custom_prefix(self):
+        resp = self.testapp.get('/', extra_environ={'SCRIPT_NAME': '/wayback'})
+        self._assert_basic_html(resp)
+        assert '<link rel="stylesheet" href="http://localhost:80/wayback/static/css/base.css"' in resp.text
         assert '/pywb' in resp.text
 
     def test_pywb_root(self):
         resp = self.testapp.get('/pywb/')
         self._assert_basic_html(resp)
-        assert '<link rel="stylesheet" href="/static/css/base.css"' in resp.text
+        assert '<link rel="stylesheet" href="http://localhost:80/static/css/base.css"' in resp.text
+        assert 'Search' in resp.text
+
+    def test_pywb_root_custom_prefix(self):
+        resp = self.testapp.get('/pywb/', extra_environ={'SCRIPT_NAME': '/wayback'})
+        self._assert_basic_html(resp)
+        assert '<link rel="stylesheet" href="http://localhost:80/wayback/static/css/base.css"' in resp.text
         assert 'Search' in resp.text
 
     def test_pywb_root_head(self):