From 22b4297fc52b1a69bd7a52a0086b7a8b2c31e6bf Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 15 May 2019 14:42:51 -0400 Subject: [PATCH] pywb: - Fix: a few broken tests due to iana.org requiring a user agent in its requests rewrite: - introduced a new JSWorkerRewriter class in order to support rewriting via wombat workers in the context of all supported worker variants via - ensured rewriter app correctly sets the static prefix wombat: - add wombat as submodule! --- .gitmodules | 3 + karma-tests/Makefile | 4 - karma-tests/dummy.html | 9 - karma-tests/karma.conf.js | 108 - karma-tests/wombat.spec.js | 225 -- pywb/apps/rewriterapp.py | 6 +- pywb/rewrite/content_rewriter.py | 8 +- pywb/rewrite/default_rewriter.py | 3 + pywb/rewrite/html_rewriter.py | 4 +- pywb/rewrite/regex_rewriters.py | 53 +- pywb/rewrite/rewrite_js_workers.py | 30 + pywb/rewrite/test/test_content_rewriter.py | 10 +- pywb/rewrite/test/test_html_rewriter.py | 2 +- pywb/rewrite/test/test_regex_rewriters.py | 32 +- pywb/rewrite/url_rewriter.py | 14 +- pywb/static/autoFetchWorker.js | 569 +-- pywb/static/autoFetchWorkerProxyMode.js | 468 +-- pywb/static/wombat.js | 4251 +------------------- pywb/static/wombatProxyMode.js | 463 +-- pywb/static/wombatWorkers.js | 19 + pywb/static/ww_rw.js | 82 - pywb/version.py | 2 +- tests/base_config_test.py | 16 + tests/test_root_coll.py | 2 +- wombat | 1 + 25 files changed, 709 insertions(+), 5675 deletions(-) create mode 100644 .gitmodules delete mode 100644 karma-tests/Makefile delete mode 100644 karma-tests/dummy.html delete mode 100644 karma-tests/karma.conf.js delete mode 100644 karma-tests/wombat.spec.js create mode 100644 pywb/rewrite/rewrite_js_workers.py create mode 100644 pywb/static/wombatWorkers.js delete mode 100644 pywb/static/ww_rw.js create mode 160000 wombat diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..5e5aaf88 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "wombat"] + path = wombat + url = https://github.com/webrecorder/wombat diff --git a/karma-tests/Makefile b/karma-tests/Makefile deleted file mode 100644 index bb80eeec..00000000 --- a/karma-tests/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -NODE_BIN_DIR=../node_modules/.bin - -test: - $(NODE_BIN_DIR)/karma start --single-run diff --git a/karma-tests/dummy.html b/karma-tests/dummy.html deleted file mode 100644 index 52e695fa..00000000 --- a/karma-tests/dummy.html +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - diff --git a/karma-tests/karma.conf.js b/karma-tests/karma.conf.js deleted file mode 100644 index 294c1b48..00000000 --- a/karma-tests/karma.conf.js +++ /dev/null @@ -1,108 +0,0 @@ -var sauceLabsConfig = { - testName: 'pywb Client Tests', -}; - -// see https://github.com/karma-runner/karma-sauce-launcher/issues/73 -if (process.env.TRAVIS_JOB_NUMBER) { - sauceLabsConfig.startConnect = false; - sauceLabsConfig.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER; -} - -var WOMBAT_JS_PATH = 'pywb/static/wombat.js'; - -var sauceLaunchers = { - sl_chrome: { - base: 'SauceLabs', - browserName: 'chrome', - }, - - sl_firefox: { - base: 'SauceLabs', - browserName: 'firefox', - }, - - sl_safari: { - base: 'SauceLabs', - browserName: 'safari', - platform: 'OS X 10.11', - version: '9.0', - }, - - sl_edge: { - base: 'SauceLabs', - browserName: 'MicrosoftEdge', - }, -}; - -var localLaunchers = { - localFirefox: { - base: 'Firefox', - }, -}; - -var customLaunchers = {}; - -if (process.env['SAUCE_USERNAME'] && process.env['SAUCE_ACCESS_KEY']) { - customLaunchers = sauceLaunchers; -} else { - console.error('Sauce Labs account details not set, ' + - 'Karma tests will be run only against local browsers.' + - 'Set SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables to ' + - 'run tests against Sauce Labs browsers'); - customLaunchers = localLaunchers; -} - -module.exports = function(config) { - config.set({ - basePath: '../', - - frameworks: ['mocha', 'chai'], - - files: [ - { - pattern: WOMBAT_JS_PATH, - watched: true, - included: false, - served: true, - }, - { - pattern: 'karma-tests/dummy.html', - included: false, - served: true, - }, - 'karma-tests/*.spec.js', - ], - - preprocessors: {}, - - reporters: ['progress'], - - port: 9876, - - colors: true, - - logLevel: config.LOG_INFO, - - autoWatch: true, - - sauceLabs: sauceLabsConfig, - - // Set extended timeouts to account for the slowness - // in connecting to remote browsers (eg. when using - // Sauce Labs) - // - // See https://oligofren.wordpress.com/2014/05/27/running-karma-tests-on-browserstack/ - captureTimeout: 3 * 60000, - browserNoActivityTimeout: 30 * 1000, - browserDisconnectTimeout: 10 * 1000, - browserDisconnectTolerance: 1, - - customLaunchers: customLaunchers, - - browsers: Object.keys(customLaunchers), - - singleRun: false, - - concurrency: Infinity - }) -}; diff --git a/karma-tests/wombat.spec.js b/karma-tests/wombat.spec.js deleted file mode 100644 index 4204389f..00000000 --- a/karma-tests/wombat.spec.js +++ /dev/null @@ -1,225 +0,0 @@ -var DEFAULT_TIMEOUT = 20000; - -// creates a new document in an