From c97a66703b534f645b79d533530a0c6590efbd9c Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 22 Dec 2021 16:15:27 -0800 Subject: [PATCH 1/5] More consistent env var setting / static path fix (#688) * template/custom env var fix: - ensure pywb.host_prefix, pywb.app_prefix and pywb.static_prefix set for all requests via prepare_env() - ensure X-Forwarded-Proto is accounted for in pywb.host_prefix - call prepare_env() in handle_request(), and also in rewriterapp (in case using a different front-end app). * update wombat to 3.3.6 (includes partial fix for #684) * bump version to 2.6.3 --- pywb/apps/frontendapp.py | 4 ++-- pywb/apps/rewriterapp.py | 25 +++++++++++++++++-------- pywb/version.py | 2 +- tests/test_integration.py | 15 ++++++++++++++- wombat | 2 +- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/pywb/apps/frontendapp.py b/pywb/apps/frontendapp.py index 3ab74bfc..956805eb 100644 --- a/pywb/apps/frontendapp.py +++ b/pywb/apps/frontendapp.py @@ -641,8 +641,8 @@ class FrontEndApp(object): urls = self.url_map.bind_to_environ(environ) try: endpoint, args = urls.match() - # store original script_name (original prefix) before modifications are made - environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '') + + self.rewriterapp.prepare_env(environ) # store original script_name (original prefix) before modifications are made environ['ORIG_SCRIPT_NAME'] = environ.get('SCRIPT_NAME') diff --git a/pywb/apps/rewriterapp.py b/pywb/apps/rewriterapp.py index ca5aa9d5..8123cf95 100644 --- a/pywb/apps/rewriterapp.py +++ b/pywb/apps/rewriterapp.py @@ -303,15 +303,24 @@ class RewriterApp(object): return resp - def render_content(self, wb_url, kwargs, environ): - wb_url = wb_url.replace('#', '%23') - wb_url = WbUrl(wb_url) + def prepare_env(self, environ): + """ setup environ path prefixes and scheme """ + if 'pywb.host_prefix' in environ: + return proto = environ.get('HTTP_X_FORWARDED_PROTO', self.force_scheme) if proto: environ['wsgi.url_scheme'] = proto + environ['pywb.host_prefix'] = self.get_host_prefix(environ) + environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '') + environ['pywb.static_prefix'] = environ['pywb.host_prefix'] + environ['pywb.app_prefix'] + '/' + self.static_prefix + + def render_content(self, wb_url, kwargs, environ): + wb_url = wb_url.replace('#', '%23') + wb_url = WbUrl(wb_url) + history_page = environ.pop('HTTP_X_WOMBAT_HISTORY_PAGE', '') if history_page: wb_url.url = history_page @@ -321,13 +330,13 @@ class RewriterApp(object): is_timegate = self._check_accept_dt(wb_url, environ) - host_prefix = self.get_host_prefix(environ) + self.prepare_env(environ) + + host_prefix = environ['pywb.host_prefix'] rel_prefix = self.get_rel_prefix(environ) full_prefix = host_prefix + rel_prefix - environ['pywb.host_prefix'] = host_prefix - pywb_static_prefix = host_prefix + environ.get('pywb.app_prefix', '') + '/' + self.static_prefix - environ['pywb.static_prefix'] = pywb_static_prefix - pywb_static_prefix += '/' + + pywb_static_prefix = environ['pywb.static_prefix'] + '/' is_proxy = ('wsgiprox.proxy_host' in environ) # if OPTIONS in proxy mode, just generate the proxy responss diff --git a/pywb/version.py b/pywb/version.py index e1e6add7..1fe20f2e 100644 --- a/pywb/version.py +++ b/pywb/version.py @@ -1,4 +1,4 @@ -__version__ = '2.6.2' +__version__ = '2.6.3' if __name__ == '__main__': print(__version__) diff --git a/tests/test_integration.py b/tests/test_integration.py index acfef645..f5abf5f0 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,12 +12,25 @@ class TestWbIntegration(BaseConfigTest): def test_home(self): resp = self.testapp.get('/') self._assert_basic_html(resp) + assert ' Date: Wed, 22 Dec 2021 16:51:02 -0800 Subject: [PATCH 2/5] quickfix: localization: ensure placeholder text also marked as localized, fixes #685 --- pywb/templates/search.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pywb/templates/search.html b/pywb/templates/search.html index 08331327..100aea35 100644 --- a/pywb/templates/search.html +++ b/pywb/templates/search.html @@ -26,7 +26,7 @@ window.wb_prefix = "{{ wb_prefix }}"; {% trans %}Search the {{ coll_title }} collection by url:{% endtrans %}
{% trans %}'Please enter a URL{% endtrans %} From de9b9310d4123b3db3a33fdfd77a106ec641e7bb Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 22 Dec 2021 17:26:45 -0800 Subject: [PATCH 3/5] Additional fixes for 2.6.3 (#689) CHANGES: update changes for 2.6.3 location rewrite: pass 'arguments' to rewrite func to guard against rewriting local 'location' in some circumstances, partial fix for #684 ci: add automated docker push on new v-* tag --- .github/workflows/release.yaml | 42 +++++++++++++++++++++++ CHANGES.rst | 16 +++++++++ pywb/rewrite/regex_rewriters.py | 3 +- pywb/rewrite/test/test_html_rewriter.py | 2 +- pywb/rewrite/test/test_regex_rewriters.py | 8 ++--- 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..0372e59e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,42 @@ +name: Publish Docker image +on: + release: + types: [published] + +jobs: + push_to_registries: + name: Build x86 and ARM Images and push to Dockerhub + runs-on: ubuntu-latest + steps: + - + name: Check out the repo + uses: actions/checkout@v2 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: webrecorder/pywb + tags: | + type=match,pattern=v-(.*),group=1 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + diff --git a/CHANGES.rst b/CHANGES.rst index ffcbc587..b745d5ca 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,19 @@ +pywb 2.6.3 changelist +~~~~~~~~~~~~~~~~~~~~~ + +* Fix false-positive rewriting of ``localion`` through additional check if local var is used, fixes `#684 `_ + +* Fix missing localization of placeholder, fixes `#685 `_ + +* Fix regression caused by 2.6.2, ensure pywb.app_prefix, pywb.host_prefix and pywb.static_prefix paths set correctly for all pages `#688 `_, fixes `#686 `_ + +* Documentation: Fixes to ``cdx-indexer`` helped (from @ldko) `#683 `_ + +* Update wombat.js to 3.3.6 + +* Add automatic Docker push on version tag + + pywb 2.6.2 changelist ~~~~~~~~~~~~~~~~~~~~~ diff --git a/pywb/rewrite/regex_rewriters.py b/pywb/rewrite/regex_rewriters.py index 6c8200ba..333b2ca4 100644 --- a/pywb/rewrite/regex_rewriters.py +++ b/pywb/rewrite/regex_rewriters.py @@ -82,7 +82,7 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj; # By using a function the expression injected is an call expression that plays nice in those cases this_rw = '_____WB$wombat$check$this$function_____(this)' - check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = ' + check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = ' self.local_objs = [ 'window', @@ -96,6 +96,7 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj; ] local_declares = '\n'.join([local_var_line.format(obj, local_init_func_name) for obj in self.local_objs]) + local_declares += "\nlet arguments;" prop_str = '|'.join(self.local_objs) diff --git a/pywb/rewrite/test/test_html_rewriter.py b/pywb/rewrite/test/test_html_rewriter.py index 5629aff9..8809441c 100644 --- a/pywb/rewrite/test/test_html_rewriter.py +++ b/pywb/rewrite/test/test_html_rewriter.py @@ -394,7 +394,7 @@ r""" # parse attr with js proxy, rewrite location assignment >>> parse('', js_proxy=True) - + # parse attr with js proxy, assigning to location.href, no location assignment rewrite needed >>> parse('', js_proxy=True) diff --git a/pywb/rewrite/test/test_regex_rewriters.py b/pywb/rewrite/test/test_regex_rewriters.py index f3aa4aaf..af08b5ee 100644 --- a/pywb/rewrite/test/test_regex_rewriters.py +++ b/pywb/rewrite/test/test_regex_rewriters.py @@ -131,13 +131,13 @@ r""" #================================================================= >>> _test_js_obj_proxy('var foo = this; location = bar') -'var foo = _____WB$wombat$check$this$function_____(this); location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar' +'var foo = _____WB$wombat$check$this$function_____(this); location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = bar' >>> _test_js_obj_proxy('var that = this\n location = bar') -'var that = _____WB$wombat$check$this$function_____(this)\n location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar' +'var that = _____WB$wombat$check$this$function_____(this)\n location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = bar' >>> _test_js_obj_proxy('location = "xyz"') -'location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = "xyz"' +'location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = "xyz"' >>> _test_js_obj_proxy('var foo = this.location') 'var foo = _____WB$wombat$check$this$function_____(this).location' @@ -213,7 +213,7 @@ r""" 'this. alocation = http://example.com/' >>> _test_js_obj_proxy(r'this. location = http://example.com/') -'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = http://example.com/' +'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = http://example.com/' >>> _test_js_obj_proxy('eval(a)') 'WB_wombat_runEval(function _____evalIsEvil(_______eval_arg$$) { return eval(_______eval_arg$$); }.bind(this)).eval(a)' From c0519a53c38915e0f9721fa6bf999f27df23a968 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 22 Dec 2021 17:38:29 -0800 Subject: [PATCH 4/5] ci release: update description --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0372e59e..b197ef4f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,7 +5,7 @@ on: jobs: push_to_registries: - name: Build x86 and ARM Images and push to Dockerhub + name: Build pywb Docker image for release and push to Dockerhub runs-on: ubuntu-latest steps: - From 7ff789f1a8e246720dab7744617824aa1e7d06c9 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 22 Dec 2021 17:42:19 -0800 Subject: [PATCH 5/5] CHANGES: fix typos in changelist --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b745d5ca..020c35ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ pywb 2.6.3 changelist ~~~~~~~~~~~~~~~~~~~~~ -* Fix false-positive rewriting of ``localion`` through additional check if local var is used, fixes `#684 `_ +* Fix false-positive rewriting of ``location`` through additional check if local var is used, fixes `#684 `_ * Fix missing localization of placeholder, fixes `#685 `_ @@ -11,7 +11,7 @@ pywb 2.6.3 changelist * Update wombat.js to 3.3.6 -* Add automatic Docker push on version tag +* Add automatic Docker push on new GitHub release pywb 2.6.2 changelist