backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

wombat overhaul! fixes #449 (#451)

Browse Source 
wombat:
 - I: function overrides applied by wombat now better appear to be the original new function name same as originals when possible
 - I: WombatLocation now looks and behaves more like the original Location interface
 - I: The custom storage class now looks and behaves more like the original Storage
 - I: SVG image rewriting has been improved: both the href and xlink:href deprecated since SVG2 now rewritten always
 - I: document.open now handles the case of creation of a new window
 - I: Request object rewriting of the readonly href property is now correctly handled
 - I: EventTarget.addEventListener, removeEventListener overrides now preserve the original this argument of the wrapped listener
 - A: document.close override to ensure wombat is initialized after write or writeln usage
 - A: reconstruction of <doctype...> in rewriteHTMLComplete IFF it was included in the original string of HTML
 - A: document.body setter override to ensure rewriting of the new body or frameset
 - A: Attr.[value, nodeValue, textContent] added setter override to perform URL rewrites
 - A: SVGElements rewriting of the filter, style, xlink:href, href, and src attributes
 - A: HTMLTrackElement rewriting of the src attribute of the
 - A: HTMLQuoteElement and HTMLModElement rewriting of the cite attribute
 - A: Worklet.addModule: Loads JS module specified by a URL.
 - A: HTMLHyperlinkElementUtils overrides to the areaelement
 - A: ShadowRootoverrides to: innerHTML even though inherites from DocumentFragement and Node it still has innerHTML getter setter.
 - A: ShadowRoot, Element, DocumentFragment append, prepend: adds strings of HTML or a new Node inherited from ParentNode
 - A: StylePropertyMap override: New way to access and set CSS properties.
 - A: Response.redirecthttps rewriting of the URL argument.
 - A:  UIEvent, MouseEvent, TouchEvent, KeyboardEvent, WheelEvent, InputEvent, and CompositionEven constructor and init{even-name} overrides in order to ensure that wombats JS Proxy usage does not affect their defined behaviors
 - A: XSLTProcessor override to ensure its usage is not affected by wombats JS Proxy usage.
 - A: navigator.unregisterProtocolHandler: Same override as existing navigator.registerProtocolHandler but from the inverse operation
 - A: PresentationRequest: Constructor takes a URL or an array of URLs.
 - A: EventSource and WebSocket override in order to ensure that they do not cause live leaks
 - A: overrides for the child node interface
 - Fix: autofetch worker creatation of the backing worker when it is operating within an execution context with a null origin
tests:
  - A: 559 tests specific to wombat and client side rewritting
pywb:
  - Fix: a few broken tests due to iana.org requiring a user agent in its requests
rewrite:
  - introduced a new JSWorkerRewriter class in order to support rewriting via wombat workers in the context of all supported worker variants via
  - ensured rewriter app correctly sets the static prefix
ci:
 - Modified travis.yml to specifically enumerate jobs
documentation:
  - Documented new wombat, wombat proxy moded, wombat workers
auto-fetch:
 - switched to mutation observer when in proxy mode so that the behaviors can operate in tandem with the autofetcher
This commit is contained in:
John Berlin 2019-05-15 14:42:51 -04:00 committed by Ilya Kreymer
parent 77f8bb6476
commit 94784d6e5d
69 changed files with 18415 additions and 5721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
.travis.yml
View File

@ -7,44 +7,55 @@ python:
- "3.7"
dist: xenial
addons:
chrome: stable
sauce_connect: true
env:
- WR_TEST=no
- WR_TEST=yes
services: xvfb
cache:
directories:
- node_modules
sudo: required
before_install:
- ./.travis/beforeInstall.sh
install:
- ./.travis/install.sh
before_install:
- 'if [ "$WR_TEST" = "yes" ]; then sudo sysctl kernel.unprivileged_userns_clone=1; fi'
script:
- ./.travis/test.sh
after_success:
- codecov
matrix:
env:
- WR_TEST=no WOMBAT_TEST=no
jobs:
allow_failures:
- env: WR_TEST=yes
exclude:
- env: WR_TEST=yes
python: "2.7"
- env: WR_TEST=yes
python: "3.5"
- env: WR_TEST=yes
- env:
- WR_TEST=yes WOMBAT_TEST=no
include:
- stage: test
name: "Replay Tests"
python: "3.7"
env:
- WR_TEST=yes WOMBAT_TEST=no
addons:
apt:
update: true
sources:
- sourceline: "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main"
key_url: https://dl-ssl.google.com/linux/linux_signing_key.pub
packages:
- google-chrome-unstable # this is canary or dev
services: xvfb
- stage: test
name: "Wombat Tests"
language: node_js
node_js: 12.0.0
env:
- WR_TEST=no WOMBAT_TEST=yes
addons:
apt:
update: true
sources:
- sourceline: "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main"
key_url: https://dl-ssl.google.com/linux/linux_signing_key.pub
packages:
- google-chrome-unstable # this is canary or dev
services: xvfb

5
.travis/beforeInstall.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
if [[ ${WR_TEST} = "yes" || ${WOMBAT_TEST} == "yes" ]]; then
sudo sysctl kernel.unprivileged_userns_clone=1
fi

31
.travis/install.sh
View File

@ -1,16 +1,23 @@
#!/bin/bash
set -e
set -ev
pip install --upgrade pip setuptools
python setup.py -q install
pip install -r extra_requirements.txt
pip install coverage pytest-cov coveralls
pip install codecov
npm install
if [[ ${WOMBAT_TEST} = "no" ]]; then
pip install --upgrade pip setuptools
python setup.py -q install
pip install -r extra_requirements.txt
pip install coverage pytest-cov coveralls
pip install codecov
if [ "$WR_TEST" = "yes" ]; then
git clone https://github.com/webrecorder/webrecorder-tests.git
cd webrecorder-tests
pip install --upgrade -r requirements.txt
./bootstrap.sh
if [[ ${WR_TEST} = "yes" ]]; then
git clone https://github.com/webrecorder/webrecorder-tests.git
cd webrecorder-tests
pip install --upgrade -r requirements.txt
./bootstrap.sh
cd ..
fi
else
cd wombat
./boostrap.sh
cd ..
fi

12
.travis/test.sh
View File

@ -1,10 +1,14 @@
#!/bin/bash
set -e
set -ev
if [ "$WR_TEST" = "no" ]; then
if [[ ${WR_TEST} = "no" && ${WOMBAT_TEST} = "no" ]]; then
python setup.py test
cd karma-tests && make test && cd ..
else
elif [[ ${WR_TEST} = "yes" && ${WOMBAT_TEST} = "no" ]]; then
cd webrecorder-tests
INTRAVIS=1 pytest -m "pywbtest and chrometest"
cd ..
elif [[ ${WR_TEST} = "no" && ${WOMBAT_TEST} = "yes" ]]; then
cd wombat
yarn run test
cd ..
fi

12
Dockerfile
View File

@ -1,19 +1,25 @@
ARG PYTHON=python:3.7.2
FROM node:11.11.0 as wombat
FROM $PYTHON
COPY ./wombat ./buildWombat
WORKDIR buildWombat
RUN yarn install && yarn run build-prod
FROM $PYTHON as pywb
WORKDIR /pywb
COPY requirements.txt extra_requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt -r extra_requirements.txt
COPY . ./
COPY --from=wombat /pywb/static/*.js ./pywb/static/
RUN python setup.py install \
&& mv ./docker-entrypoint.sh / \
&& mkdir /uwsgi && mv ./uwsgi.ini /uwsgi/ \
&& mkdir /webarchive && mv ./config.yaml /webarchive/
&& mkdir /webarchive && mv ./config.yaml /webarchive/ \
&& rm -rf ./wombat
WORKDIR /webarchive

4
karma-tests/Makefile
View File

@ -1,4 +0,0 @@
NODE_BIN_DIR=../node_modules/.bin
test:
$(NODE_BIN_DIR)/karma start --single-run

9
karma-tests/dummy.html
View File

@ -1,9 +0,0 @@
<html>
<head><meta charset="UTF-8"></head>
<body>
<!-- This is a dummy page used in
tests of Wombat's live-rewriting
functionality.
!-->
</body>
</html>

108
karma-tests/karma.conf.js
View File

@ -1,108 +0,0 @@
var sauceLabsConfig = {
testName: 'pywb Client Tests',
};
// see https://github.com/karma-runner/karma-sauce-launcher/issues/73
if (process.env.TRAVIS_JOB_NUMBER) {
sauceLabsConfig.startConnect = false;
sauceLabsConfig.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER;
}
var WOMBAT_JS_PATH = 'pywb/static/wombat.js';
var sauceLaunchers = {
sl_chrome: {
base: 'SauceLabs',
browserName: 'chrome',
},
sl_firefox: {
base: 'SauceLabs',
browserName: 'firefox',
},
sl_safari: {
base: 'SauceLabs',
browserName: 'safari',
platform: 'OS X 10.11',
version: '9.0',
},
sl_edge: {
base: 'SauceLabs',
browserName: 'MicrosoftEdge',
},
};
var localLaunchers = {
localFirefox: {
base: 'Firefox',
},
};
var customLaunchers = {};
if (process.env['SAUCE_USERNAME'] && process.env['SAUCE_ACCESS_KEY']) {
customLaunchers = sauceLaunchers;
} else {
console.error('Sauce Labs account details not set, ' +
'Karma tests will be run only against local browsers.' +
'Set SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables to ' +
'run tests against Sauce Labs browsers');
customLaunchers = localLaunchers;
}
module.exports = function(config) {
config.set({
basePath: '../',
frameworks: ['mocha', 'chai'],
files: [
{
pattern: WOMBAT_JS_PATH,
watched: true,
included: false,
served: true,
},
{
pattern: 'karma-tests/dummy.html',
included: false,
served: true,
},
'karma-tests/*.spec.js',
],
preprocessors: {},
reporters: ['progress'],
port: 9876,
colors: true,
logLevel: config.LOG_INFO,
autoWatch: true,
sauceLabs: sauceLabsConfig,
// Set extended timeouts to account for the slowness
// in connecting to remote browsers (eg. when using
// Sauce Labs)
//
// See https://oligofren.wordpress.com/2014/05/27/running-karma-tests-on-browserstack/
captureTimeout: 3 * 60000,
browserNoActivityTimeout: 30 * 1000,
browserDisconnectTimeout: 10 * 1000,
browserDisconnectTolerance: 1,
customLaunchers: customLaunchers,
browsers: Object.keys(customLaunchers),
singleRun: false,
concurrency: Infinity
})
};

225
karma-tests/wombat.spec.js
View File

@ -1,225 +0,0 @@
var DEFAULT_TIMEOUT = 20000;
// creates a new document in an <iframe> and runs
// a WombatJS test case in it.
//
// A new <iframe> is used for each test so that each
// case is run with fresh Document and Window objects,
// since Wombat monkey-patches many Document and Window
// functions
//
function runWombatTest(testCase, done) {
// create an <iframe>
var testFrame = document.createElement('iframe');
testFrame.src = '/base/karma-tests/dummy.html';
document.body.appendChild(testFrame);
testFrame.contentWindow.addEventListener('load', function () {
var testDocument = testFrame.contentDocument;
function runFunctionInIFrame(func) {
testFrame.contentWindow.eval('(' + func.toString() + ')()');
}
// expose an error reporting function to the <iframe>
window.reportError = function(ex) {
done(new Error(ex));
};
// expose utility methods for assertion testing in tests.
// (We used to expose chai asserts here but Karma's default
// error reporter replaces URLs in exception messages with
// the corresponding file paths, which is unhelpful for us
// since assert.equal() will often be called with URLs in our tests)
window.assert = {
equal: function (a, b) {
if (a !== b) {
console.error('Mismatch between', a, 'and', b);
throw new Error('AssertionError');
}
}
};
runFunctionInIFrame(function () {
// re-assign the iframe's console object to the parent window's
// console so that messages are intercepted by Karma
// and output to wherever it is configured to send
// console logs (typically stdout)
console = window.parent.console;
window.onerror = function (message, url, line, col, error) {
if (error) {
console.log(error.stack);
}
reportError(new Error(message));
};
// expose chai's assertion testing API to the test script
window.assert = window.parent.assert;
window.reportError = window.parent.reportError;
// helpers which check whether DOM property overrides are supported
// in the current browser
window.domTests = {
areDOMPropertiesConfigurable: function () {
var descriptor = Object.getOwnPropertyDescriptor(Node.prototype, 'baseURI');
if (descriptor && !descriptor.configurable) {
return false;
} else {
return true;
}
}
};
});
try {
runFunctionInIFrame(testCase.initScript);
} catch (e) {
throw new Error('Configuring Wombat failed: ' + e.toString());
}
try {
testFrame.contentWindow.eval(testCase.wombatScript);
runFunctionInIFrame(function () {
new window._WBWombat(window, wbinfo);
});
} catch (e) {
console.error(e.stack);
throw new Error('Initializing WombatJS failed: ' + e.toString());
}
if (testCase.html) {
testDocument.body.innerHTML = testCase.html;
}
if (testCase.testScript) {
try {
runFunctionInIFrame(testCase.testScript);
} catch (e) {
throw new Error('Test script failed: ' + e.toString());
}
}
testFrame.remove();
done();
});
}
describe('WombatJS', function () {
this.timeout(DEFAULT_TIMEOUT);
var wombatScript;
before(function (done) {
// load the source of the WombatJS content
// rewriting script
var req = new XMLHttpRequest();
req.open('GET', '/base/pywb/static/wombat.js');
req.onload = function () {
wombatScript = req.responseText;
done();
};
req.send();
});
it('should load', function (done) {
runWombatTest({
initScript: function () {
wbinfo = {
wombat_opts: {},
wombat_ts: '',
is_live: false,
top_url: ''
};
},
wombatScript: wombatScript,
}, done);
});
describe('anchor rewriting', function () {
var config;
beforeEach(function () {
config = {
initScript: function () {
wbinfo = {
wombat_opts: {},
wombat_scheme: 'http',
prefix: window.location.origin,
wombat_ts: '',
is_live: false,
top_url: ''
};
},
wombatScript: wombatScript,
html: '<a href="foobar.html" id="link">A link</a>',
};
});
it('should rewrite links in dynamically injected <a> tags', function (done) {
config.testScript = function () {
if (domTests.areDOMPropertiesConfigurable()) {
var link = document.getElementById('link');
assert.equal(link.href, 'http:///base/karma-tests/foobar.html');
}
};
runWombatTest(config, done);
});
it('toString() should return the rewritten URL', function (done) {
config.testScript = function () {
if (domTests.areDOMPropertiesConfigurable()) {
var link = document.getElementById('link');
assert.equal(link.href, link.toString());
}
};
runWombatTest(config, done);
});
});
describe('base URL overrides', function () {
it('document.baseURI should return the original URL', function (done) {
runWombatTest({
initScript: function () {
wbinfo = {
wombat_opts: {},
prefix: window.location.origin,
wombat_ts: '',
wombat_scheme: 'http',
is_live: false,
top_url: ''
};
},
wombatScript: wombatScript,
testScript: function () {
var baseURI = document.baseURI;
if (typeof baseURI !== 'string') {
throw new Error('baseURI is not a string');
}
if (domTests.areDOMPropertiesConfigurable()) {
assert.equal(baseURI, 'http:///base/karma-tests/dummy.html');
}
},
}, done);
});
it('should allow base.href to be assigned', function (done) {
runWombatTest({
initScript: function () {
wbinfo = {
wombat_opts: {},
wombat_scheme: 'http',
is_live: false,
top_url: ''
};
},
wombatScript: wombatScript,
testScript: function () {
'use strict';
var baseElement = document.createElement('base');
baseElement.href = 'http://foobar.com/base';
assert.equal(baseElement.href, 'http://foobar.com/base');
},
}, done);
});
});
});

6
pywb/apps/rewriterapp.py
View File

@ -238,7 +238,8 @@ class RewriterApp(object):
host_prefix = self.get_host_prefix(environ)
rel_prefix = self.get_rel_prefix(environ)
full_prefix = host_prefix + rel_prefix
pywb_static_prefix = environ.get('pywb.host_prefix', '') + environ.get('pywb.app_prefix', '') + environ.get(
'pywb.static_prefix', '/static/')
is_proxy = ('wsgiprox.proxy_host' in environ)
response = self.handle_custom_response(environ, wb_url,
@ -257,7 +258,8 @@ class RewriterApp(object):
urlrewriter = UrlRewriter(wb_url,
prefix=full_prefix,
full_prefix=full_prefix,
rel_prefix=rel_prefix)
rel_prefix=rel_prefix,
pywb_static_prefix=pywb_static_prefix)
framed_replay = self.framed_replay

8
pywb/rewrite/content_rewriter.py
View File

@ -15,6 +15,8 @@ from pywb.utils.io import StreamIter, BUFF_SIZE
from pywb.utils.loaders import load_yaml_config, load_py_name
WORKER_MODS = {"wkr_", "sw_"} # type: Set[str]
# ============================================================================
class BaseContentRewriter(object):
@ -423,8 +425,8 @@ class RewriteInfo(object):
def _resolve_text_type(self, text_type):
mod = self.url_rewriter.wburl.mod
if mod == 'sw_' or mod == 'wkr_':
return None
if mod in WORKER_MODS:
return 'js-worker'
if text_type == 'css' and mod == 'js_':
text_type = 'css'
@ -495,7 +497,7 @@ class RewriteInfo(object):
return True
def is_url_rw(self):
if self.url_rewriter.wburl.mod in ('id_', 'bn_', 'sw_', 'wkr_'):
if self.url_rewriter.wburl.mod in ('id_', 'bn_', 'wkrf_'):
return False
return True

3
pywb/rewrite/default_rewriter.py
View File

@ -15,6 +15,8 @@ from pywb.rewrite.rewrite_dash import RewriteDASH
from pywb.rewrite.rewrite_hls import RewriteHLS
from pywb.rewrite.rewrite_amf import RewriteAMF
from pywb.rewrite.rewrite_js_workers import JSWorkerRewriter
from pywb import DEFAULT_RULES_FILE
import copy
@ -34,6 +36,7 @@ class DefaultRewriter(BaseContentRewriter):
'js': JSLocationOnlyRewriter,
'js-proxy': JSNoneRewriter,
'js-worker': JSWorkerRewriter,
'json': JSONPRewriter,

4
pywb/rewrite/html_rewriter.py
View File

@ -58,7 +58,7 @@ class HTMLRewriterMixin(StreamingRewriter):
'embed': {'src': 'oe_'},
'head': {'': defmod}, # for head rewriting
'iframe': {'src': 'if_'},
'image': {'src': 'im_', 'xlink:href': 'im_'},
'image': {'src': 'im_', 'xlink:href': 'im_', 'href': 'im_'},
'img': {'src': 'im_',
'srcset': 'im_'},
'ins': {'cite': defmod},
@ -74,7 +74,7 @@ class HTMLRewriterMixin(StreamingRewriter):
'q': {'cite': defmod},
'ref': {'href': 'oe_'},
'script': {'src': 'js_', 'xlink:href': 'js_'}, # covers both HTML and SVG script tags
'source': {'src': 'oe_'},
'source': {'src': 'oe_', 'srcset': 'oe_'},
'video': {'src': 'oe_',
'poster': 'im_'},
}

53
pywb/rewrite/regex_rewriters.py
View File

@ -63,48 +63,59 @@ class RxRules(object):
class JSWombatProxyRules(RxRules):
def __init__(self):
local_init_func = '\nvar {0} = function(name) {{\
return (self._wb_wombat && self._wb_wombat.local_init &&\
return (self._wb_wombat && self._wb_wombat.local_init && \
self._wb_wombat.local_init(name)) || self[name]; }};\n\
if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ return obj; }} }}\n\
{{\n'
local_check_this_fn = 'var {0} = function (thisObj) {{ \
if (thisObj && thisObj._WB_wombat_obj_proxy) return thisObj._WB_wombat_obj_proxy; return thisObj; }};'
local_init_func_name = '_____WB$wombat$assign$function_____'
local_var_line = 'let {0} = {1}("{0}");'
this_rw = '(this && this._WB_wombat_obj_proxy || this)'
local_check_this_func_name = '_____WB$wombat$check$this$function_____'
check_loc = '(self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = '
# we must use a function to perform the this check because most minfiers reduce the number of statements
# by turning everything into one or more expressions. Our previous rewrite was an logical expression,
# (this && this._WB_wombat_obj_proxy || this), that would cause the outer expression to be invalid when
# it was used as the LHS of certain expressions.
# e.g. assignment expressions containing non parenthesized logical expression.
# By using a function the expression injected is an call expression that plays nice in those cases
this_rw = '_____WB$wombat$check$this$function_____(this)'
check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = '
self.local_objs = [
'window',
'self',
'document',
'location',
'top',
'parent',
'frames',
'opener']
'window',
'self',
'document',
'location',
'top',
'parent',
'frames',
'opener'
]
local_declares = '\n'.join([local_var_line.format(obj, local_init_func_name) for obj in self.local_objs])
prop_str = '|'.join(self.local_objs)
rules = [
(r'(?<=\.)postMessage\b\(', self.add_prefix('__WB_pmw(self).'), 0),
(r'(?<![$.])\s*location\b\s*[=]\s*(?![=])', self.add_suffix(check_loc), 0),
(r'\breturn\s+this\b\s*(?![.$])', self.replace_str(this_rw), 0),
(r'(?<=[\n])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(';' + this_rw), 0),
(r'(?<![$.])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(this_rw), 0),
(r'(?<=[=])\s*this\b\s*(?![.$])', self.replace_str(this_rw), 0),
('\}(?:\s*\))?\s*\(this\)', self.replace_str(this_rw), 0),
(r'(?<=[^|&][|&]{2})\s*this\b\s*(?![|&.$]([^|&]|$))', self.replace_str(this_rw), 0),
(r'(?<=\.)postMessage\b\(', self.add_prefix('__WB_pmw(self).'), 0),
(r'(?<![$.])\s*location\b\s*[=]\s*(?![=])', self.add_suffix(check_loc), 0),
(r'\breturn\s+this\b\s*(?![.$])', self.replace_str(this_rw), 0),
(r'(?<=[\n])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(';' + this_rw), 0),
(r'(?<![$.])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(this_rw), 0),
(r'(?<=[=])\s*this\b\s*(?![.$])', self.replace_str(this_rw), 0),
('\}(?:\s*\))?\s*\(this\)', self.replace_str(this_rw), 0),
(r'(?<=[^|&][|&]{2})\s*this\b\s*(?![|&.$]([^|&]|$))', self.replace_str(this_rw), 0),
]
super(JSWombatProxyRules, self).__init__(rules)
self.first_buff = local_init_func.format(local_init_func_name) + local_declares
self.first_buff = local_check_this_fn.format(local_check_this_func_name) + local_init_func.format(
local_init_func_name) + local_declares + '\n\n'
self.last_buff = '\n\n}'

30
pywb/rewrite/rewrite_js_workers.py Normal file
View File

@ -0,0 +1,30 @@
from pywb.rewrite.content_rewriter import StreamingRewriter, WORKER_MODS
__all__ = ["JSWorkerRewriter"]
INJECT = "(function() { self.importScripts('%s'); new WBWombat(%s); })();"
INIT = "{'prefix': '%s', 'prefixMod': '%s/', 'originalURL': '%s'}"
class JSWorkerRewriter(StreamingRewriter):
"""A simple rewriter for rewriting web or service workers.
The only rewriting that occurs is the injection of the init code
for wombatWorkers.js.
This allows for all them to operate as expected on the live web.
"""
def __init__(self, url_rewriter, align_to_line=True, first_buff=''):
"""Initialize a new JSWorkerRewriter
:param UrlRewriter url_rewriter: The url rewriter for this rewrite
:param bool align_to_line: Should the response stream be aliened to line boundaries
:param str first_buff: The first string to be added to the rewrite
:rtype: None
"""
super(JSWorkerRewriter, self).__init__(url_rewriter, align_to_line, first_buff)
wb_url = self.url_rewriter.wburl
if wb_url.mod in WORKER_MODS:
rw_url = self.url_rewriter.pywb_static_prefix + "wombatWorkers.js"
prefix = self.url_rewriter.full_prefix
init = INIT % (prefix, prefix + 'wkrf_', wb_url.url)
self.first_buff = INJECT % (rw_url, init)

10
pywb/rewrite/test/test_content_rewriter.py
View File

@ -235,24 +235,22 @@ class TestContentRewriter(object):
def test_rewrite_sw_add_headers(self):
headers = {'Content-Type': 'application/x-javascript'}
content = 'function() { location.href = "http://example.com/"; }'
content = "function() { location.href = 'http://example.com/'; }"
headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701sw_')
assert ('Content-Type', 'application/x-javascript') in headers.headers
assert ('Service-Worker-Allowed', 'http://localhost:8080/prefix/201701mp_/http://example.com/') in headers.headers
exp = 'function() { location.href = "http://example.com/"; }'
assert b''.join(gen).decode('utf-8') == exp
assert "self.importScripts('wombatWorkers.js');" in b''.join(gen).decode('utf-8')
def test_rewrite_worker(self):
headers = {'Content-Type': 'application/x-javascript'}
content = 'importScripts("http://example.com/js.js")'
content = "importScripts('http://example.com/js.js')"
rwheaders, gen, is_rw = self.rewrite_record(headers, content, ts='201701wkr_')
exp = 'importScripts("http://example.com/js.js")'
assert b''.join(gen).decode('utf-8') == exp
assert "self.importScripts('wombatWorkers.js');" in b''.join(gen).decode('utf-8')
def test_banner_only_no_cookie_rewrite(self):
headers = {'Set-Cookie': 'foo=bar; Expires=Wed, 13 Jan 2021 22:23:01 GMT; Path=/',

2
pywb/rewrite/test/test_html_rewriter.py
View File

@ -389,7 +389,7 @@ r"""
# parse attr with js proxy, rewrite location assignment
>>> parse('<html><a href="javascript:location=\'foo.html\'"></a></html>', js_proxy=True)
<html><a href="javascript:{ location=(self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = 'foo.html' }"></a></html>
<html><a href="javascript:{ location=((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = 'foo.html' }"></a></html>
# parse attr with js proxy, assigning to location.href, no location assignment rewrite needed
>>> parse('<html><a href="javascript:location.href=\'foo.html\'"></a></html>', js_proxy=True)

32
pywb/rewrite/test/test_regex_rewriters.py
View File

@ -131,49 +131,49 @@ r"""
#=================================================================
>>> _test_js_obj_proxy('var foo = this; location = bar')
'var foo = (this && this._WB_wombat_obj_proxy || this); location = (self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = bar'
'var foo = _____WB$wombat$check$this$function_____(this); location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar'
>>> _test_js_obj_proxy('var that = this\n location = bar')
'var that = (this && this._WB_wombat_obj_proxy || this)\n location = (self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = bar'
'var that = _____WB$wombat$check$this$function_____(this)\n location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = bar'
>>> _test_js_obj_proxy('location = "xyz"')
'location = (self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = "xyz"'
'location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = "xyz"'
>>> _test_js_obj_proxy('var foo = this.location')
'var foo = (this && this._WB_wombat_obj_proxy || this).location'
'var foo = _____WB$wombat$check$this$function_____(this).location'
>>> _test_js_obj_proxy('A = B\nthis.location = "foo"')
'A = B\n;(this && this._WB_wombat_obj_proxy || this).location = "foo"'
'A = B\n;_____WB$wombat$check$this$function_____(this).location = "foo"'
>>> _test_js_obj_proxy('var foo = this.location2')
'var foo = this.location2'
>>> _test_js_obj_proxy('func(Function("return this"));')
'func(Function("return (this && this._WB_wombat_obj_proxy || this)"));'
'func(Function("return _____WB$wombat$check$this$function_____(this)"));'
>>> _test_js_obj_proxy('A.call(function() { return this });')
'A.call(function() { return (this && this._WB_wombat_obj_proxy || this) });'
>>> _test_js_obj_proxy('A.call(function() { return this });')
'A.call(function() { return _____WB$wombat$check$this$function_____(this) });'
>>> _test_js_obj_proxy('this.document.location = foo')
'(this && this._WB_wombat_obj_proxy || this).document.location = foo'
'_____WB$wombat$check$this$function_____(this).document.location = foo'
>>> _test_js_obj_proxy('if (that != this) { ... }')
'if (that != (this && this._WB_wombat_obj_proxy || this)) { ... }'
'if (that != _____WB$wombat$check$this$function_____(this)) { ... }'
>>> _test_js_obj_proxy('function(){...} (this)')
'function(){...} ((this && this._WB_wombat_obj_proxy || this))'
'function(){...} (_____WB$wombat$check$this$function_____(this))'
>>> _test_js_obj_proxy('function(){...} ) (this); foo(this)')
'function(){...} ) ((this && this._WB_wombat_obj_proxy || this)); foo(this)'
'function(){...} ) (_____WB$wombat$check$this$function_____(this)); foo(this)'
>>> _test_js_obj_proxy('var foo = that || this ;')
'var foo = that || (this && this._WB_wombat_obj_proxy || this) ;'
'var foo = that || _____WB$wombat$check$this$function_____(this) ;'
>>> _test_js_obj_proxy('a||this||that')
'a||(this && this._WB_wombat_obj_proxy || this)||that'
'a||_____WB$wombat$check$this$function_____(this)||that'
>>> _test_js_obj_proxy('a||this)')
'a||(this && this._WB_wombat_obj_proxy || this))'
'a||_____WB$wombat$check$this$function_____(this))'
# not rewritten
>>> _test_js_obj_proxy('var window = this$')
@ -207,7 +207,7 @@ r"""
'this. alocation = http://example.com/'
>>> _test_js_obj_proxy(r'this. location = http://example.com/')
'this. location = (self.__WB_check_loc && self.__WB_check_loc(location) || {}).href = http://example.com/'
'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location)) || {}).href = http://example.com/'

14
pywb/rewrite/url_rewriter.py
View File

@ -23,7 +23,7 @@ class UrlRewriter(object):
REL_PATH = '/'
def __init__(self, wburl, prefix='', full_prefix=None, rel_prefix=None,
root_path=None, cookie_scope=None, rewrite_opts=None):
root_path=None, cookie_scope=None, rewrite_opts=None, pywb_static_prefix=None):
self.wburl = wburl if isinstance(wburl, WbUrl) else WbUrl(wburl)
self.prefix = prefix
self.full_prefix = full_prefix or prefix
@ -36,10 +36,22 @@ class UrlRewriter(object):
self.prefix_abs = self.prefix and self.prefix.startswith(self.PROTOCOLS)
self.cookie_scope = cookie_scope
self.rewrite_opts = rewrite_opts or {}
self._pywb_static_prefix = pywb_static_prefix
if self.rewrite_opts.get('punycode_links'):
self.wburl._do_percent_encode = False
@property
def pywb_static_prefix(self):
"""Returns the static path URL
:rtype: str
"""
if self._pywb_static_prefix is None:
return ''
if self._pywb_static_prefix.startswith(self.PROTOCOLS):
return self._pywb_static_prefix
return self.urljoin(self.full_prefix, self._pywb_static_prefix)
def rewrite(self, url, mod=None, force_abs=False):
# if special protocol, no rewriting at all
if url.startswith(self.NO_REWRITE_URI_PREFIX):

569
pywb/static/autoFetchWorker.js
View File

@ -15,338 +15,355 @@ var autofetcher = null;
function noop() {}
if (typeof self.Promise === 'undefined') {
// not kewl we must polyfill Promise
self.Promise = function (executor) {
executor(noop, noop);
};
self.Promise.prototype.then = function (cb) {
if (cb) cb();
return this;
};
self.Promise.prototype.catch = function () {
return this;
};
self.Promise.all = function (values) {
return new Promise(noop);
};
// not kewl we must polyfill Promise
self.Promise = function(executor) {
executor(noop, noop);
};
self.Promise.prototype.then = function(cb) {
if (cb) cb();
return this;
};
self.Promise.prototype.catch = function() {
return this;
};
self.Promise.all = function(values) {
return new Promise(noop);
};
}
if (typeof self.fetch === 'undefined') {
// not kewl we must polyfill fetch.
self.fetch = function (url) {
return new Promise(function (resolve) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.send();
resolve();
});
};
// not kewl we must polyfill fetch.
self.fetch = function(url) {
return new Promise(function(resolve) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.send();
resolve();
});
};
}
self.onmessage = function (event) {
var data = event.data;
switch (data.type) {
case 'values':
autofetcher.autoFetch(data);
break;
}
self.onmessage = function(event) {
var data = event.data;
switch (data.type) {
case 'values':
autofetcher.autoFetch(data);
break;
}
};
function AutoFetcher(init) {
if (!(this instanceof AutoFetcher)) {
return new AutoFetcher(init);
}
this.prefix = init.prefix;
this.mod = init.mod;
this.prefixMod = init.prefix + init.mod;
this.rwRe = new RegExp(init.rwRe);
// relative url, WorkerLocation is set by owning document
this.relative = init.prefix.split(location.origin)[1];
// schemeless url
this.schemeless = '/' + this.relative;
// local cache of URLs fetched, to reduce server load
this.seen = {};
// array of URLs to be fetched
this.queue = [];
this.avQueue = [];
// should we queue a URL or not
this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this);
this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
if (!(this instanceof AutoFetcher)) {
return new AutoFetcher(init);
}
this.prefix = init.prefix;
this.mod = init.mod;
this.prefixMod = init.prefix + init.mod;
this.rwRe = new RegExp(init.rwRe);
// relative url, WorkerLocation is set by owning document
this.relative = init.prefix.split(location.origin)[1];
// schemeless url
this.schemeless = '/' + this.relative;
// local cache of URLs fetched, to reduce server load
this.seen = {};
// array of URLs to be fetched
this.queue = [];
this.avQueue = [];
// should we queue a URL or not
this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this);
this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
}
AutoFetcher.prototype.delay = function () {
// 2 second delay seem reasonable
return new Promise(function (resolve, reject) {
setTimeout(resolve, 2000);
AutoFetcher.prototype.delay = function() {
// 2 second delay seem reasonable
return new Promise(function(resolve, reject) {
setTimeout(resolve, 2000);
});
};
AutoFetcher.prototype.imgFetchDone = function() {
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function() {
autofetcher.queuing = false;
autofetcher.fetchImgs();
});
} else {
this.queuing = false;
}
};
AutoFetcher.prototype.imgFetchDone = function () {
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuing = false;
autofetcher.fetchImgs();
});
} else {
this.queuing = false;
}
AutoFetcher.prototype.avFetchDone = function() {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function() {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
}
};
AutoFetcher.prototype.avFetchDone = function () {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
AutoFetcher.prototype.fetchAV = function() {
if (this.queuingAV || this.avQueue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (
this.avQueue.length > 0 &&
runningFetchers.length <= DefaultNumAvFetches
) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop));
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop));
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
};
AutoFetcher.prototype.fetchAV = function () {
if (this.queuingAV || this.avQueue.length === 0) {
return;
AutoFetcher.prototype.fetchImgs = function() {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch
this.queuing = true;
var runningFetchers = [];
while (
this.queue.length > 0 &&
runningFetchers.length <= DefaultNumImFetches
) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop));
}
if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop));
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (this.avQueue.length > 0 && runningFetchers.length <= DefaultNumAvFetches) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
}
Promise.all(runningFetchers)
.then(this.imgFetchDone)
.catch(this.imgFetchDone);
};
AutoFetcher.prototype.fetchImgs = function () {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch
this.queuing = true;
var runningFetchers = [];
while (this.queue.length > 0 && runningFetchers.length <= DefaultNumImFetches) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.imgFetchDone)
.catch(this.imgFetchDone);
AutoFetcher.prototype.queueNonAVURL = function(url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.queue.push(url);
};
AutoFetcher.prototype.queueNonAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.queue.push(url);
AutoFetcher.prototype.queueAVURL = function(url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.queueAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.maybeResolveURL = function (url, base) {
// given a url and base url returns a resolved full URL or
// null if resolution was unsuccessful
try {
var _url = new URL(url, base);
return _url.href;
} catch (e) {
return null;
}
};
AutoFetcher.prototype.maybeFixUpRelSchemelessPrefix = function (url) {
// attempt to ensure rewritten relative or schemeless URLs become full URLS!
// otherwise returns null if this did not happen
if (url.indexOf(this.relative) === 0) {
return url.replace(this.relative, this.prefix);
}
if (url.indexOf(this.schemeless) === 0) {
return url.replace(this.schemeless, this.prefix);
}
AutoFetcher.prototype.maybeResolveURL = function(url, base) {
// given a url and base url returns a resolved full URL or
// null if resolution was unsuccessful
try {
var _url = new URL(url, base);
return _url.href;
} catch (e) {
return null;
}
};
AutoFetcher.prototype.maybeFixUpURL = function (url, resolveOpts) {
// attempt to fix up the url and do our best to ensure we can get dat 200 OK!
if (this.rwRe.test(url)) {
return url;
}
var mod = resolveOpts.mod || 'mp_';
// first check for / (relative) or // (schemeless) rewritten urls
var maybeFixed = this.maybeFixUpRelSchemelessPrefix(url);
AutoFetcher.prototype.maybeFixUpRelSchemelessPrefix = function(url) {
// attempt to ensure rewritten relative or schemeless URLs become full URLS!
// otherwise returns null if this did not happen
if (url.indexOf(this.relative) === 0) {
return url.replace(this.relative, this.prefix);
}
if (url.indexOf(this.schemeless) === 0) {
return url.replace(this.schemeless, this.prefix);
}
return null;
};
AutoFetcher.prototype.maybeFixUpURL = function(url, resolveOpts) {
// attempt to fix up the url and do our best to ensure we can get dat 200 OK!
if (this.rwRe.test(url)) {
return url;
}
var mod = resolveOpts.mod || 'mp_';
// first check for / (relative) or // (schemeless) rewritten urls
var maybeFixed = this.maybeFixUpRelSchemelessPrefix(url);
if (maybeFixed != null) {
return maybeFixed;
}
// resolve URL against tag src
if (resolveOpts.tagSrc != null) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.tagSrc);
if (maybeFixed != null) {
return maybeFixed;
return this.prefix + mod + '/' + maybeFixed;
}
// resolve URL against tag src
if (resolveOpts.tagSrc != null) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.tagSrc);
if (maybeFixed != null) {
return this.prefix + mod + '/' + maybeFixed;
}
}
// finally last attempt resolve the originating documents base URI
if (resolveOpts.docBaseURI) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.docBaseURI);
if (maybeFixed != null) {
return this.prefix + mod + '/' + maybeFixed;
}
// finally last attempt resolve the originating documents base URI
if (resolveOpts.docBaseURI) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.docBaseURI);
if (maybeFixed != null) {
return this.prefix + mod + '/' + maybeFixed;
}
}
// not much to do now.....
return this.prefixMod + '/' + url;
}
// not much to do now.....
return this.prefixMod + '/' + url;
};
AutoFetcher.prototype.urlExtractor = function (match, n1, n2, n3, offset, string) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
this.queueNonAVURL(n2);
return n1 + n2 + n3;
AutoFetcher.prototype.urlExtractor = function(
match,
n1,
n2,
n3,
offset,
string
) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
this.queueNonAVURL(n2);
return n1 + n2 + n3;
};
AutoFetcher.prototype.handleMedia = function (mediaRules) {
// this is a broken down rewrite_style
if (mediaRules == null || mediaRules.length === 0) return;
// var rules = mediaRules.values;
for (var i = 0; i < mediaRules.length; i++) {
mediaRules[i]
.replace(STYLE_REGEX, this.urlExtractor)
.replace(IMPORT_REGEX, this.urlExtractor);
}
AutoFetcher.prototype.handleMedia = function(mediaRules) {
// this is a broken down rewrite_style
if (mediaRules == null || mediaRules.length === 0) return;
// var rules = mediaRules.values;
for (var i = 0; i < mediaRules.length; i++) {
mediaRules[i]
.replace(STYLE_REGEX, this.urlExtractor)
.replace(IMPORT_REGEX, this.urlExtractor);
}
};
AutoFetcher.prototype.handleSrc = function (srcValues, context) {
var resolveOpts = { 'docBaseURI': context.docBaseURI };
if (srcValues.value) {
resolveOpts.mod = srcValues.mod;
if (resolveOpts.mod === 1) {
return this.queueNonAVURL(this.maybeFixUpURL(srcValues.value.trim(), resolveOpts));
}
return this.queueAVURL(this.maybeFixUpURL(srcValues.value.trim(), resolveOpts));
AutoFetcher.prototype.handleSrc = function(srcValues, context) {
var resolveOpts = { docBaseURI: context.docBaseURI };
if (srcValues.value) {
resolveOpts.mod = srcValues.mod;
if (resolveOpts.mod === 1) {
return this.queueNonAVURL(
this.maybeFixUpURL(srcValues.value.trim(), resolveOpts)
);
}
var len = srcValues.values.length;
for (var i = 0; i < len; i++) {
var value = srcValues.values[i];
resolveOpts.mod = value.mod;
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(this.maybeFixUpURL(value.src, resolveOpts));
} else {
this.queueAVURL(this.maybeFixUpURL(value.src, resolveOpts));
}
return this.queueAVURL(
this.maybeFixUpURL(srcValues.value.trim(), resolveOpts)
);
}
var len = srcValues.values.length;
for (var i = 0; i < len; i++) {
var value = srcValues.values[i];
resolveOpts.mod = value.mod;
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(this.maybeFixUpURL(value.src, resolveOpts));
} else {
this.queueAVURL(this.maybeFixUpURL(value.src, resolveOpts));
}
}
};
AutoFetcher.prototype.extractSrcSetNotPreSplit = function (ssV, resolveOpts) {
// was from extract from local doc so we need to duplicate work
var srcsetValues = ssV.split(srcsetSplit);
for (var i = 0; i < srcsetValues.length; i++) {
// grab the URL not width/height key
if (srcsetValues[i]) {
var value = srcsetValues[i].trim().split(' ')[0];
var maybeResolvedURL = this.maybeFixUpURL(value.trim(), resolveOpts);
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(maybeResolvedURL);
} else {
this.queueAVURL(maybeResolvedURL);
}
}
AutoFetcher.prototype.extractSrcSetNotPreSplit = function(ssV, resolveOpts) {
if (!ssV) return;
// was from extract from local doc so we need to duplicate work
var srcsetValues = ssV.split(srcsetSplit);
for (var i = 0; i < srcsetValues.length; i++) {
// grab the URL not width/height key
if (srcsetValues[i]) {
var value = srcsetValues[i].trim().split(' ')[0];
var maybeResolvedURL = this.maybeFixUpURL(value.trim(), resolveOpts);
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(maybeResolvedURL);
} else {
this.queueAVURL(maybeResolvedURL);
}
}
}
};
AutoFetcher.prototype.extractSrcset = function (srcsets, context) {
// was rewrite_srcset and only need to q
for (var i = 0; i < srcsets.length; i++) {
// grab the URL not width/height key
var url = srcsets[i].split(' ')[0];
if (context.mod === 'im_') {
this.queueNonAVURL(url);
} else {
this.queueAVURL(url);
}
AutoFetcher.prototype.extractSrcset = function(srcsets, context) {
// was rewrite_srcset and only need to q
for (var i = 0; i < srcsets.length; i++) {
// grab the URL not width/height key
var url = srcsets[i].split(' ')[0];
if (context.mod === 'im_') {
this.queueNonAVURL(url);
} else {
this.queueAVURL(url);
}
}
};
AutoFetcher.prototype.handleSrcset = function (srcset, context) {
var resolveOpts = { 'docBaseURI': context.docBaseURI };
if (srcset.value) {
// we have a single value, this srcset came from either
// preserveDataSrcset (not presplit) preserveSrcset (presplit)
resolveOpts.mod = srcset.mod;
if (!srcset.presplit) {
// extract URLs from the srcset string
return this.extractSrcSetNotPreSplit(srcset.value, resolveOpts);
}
// we have an array of srcset URL strings
return this.extractSrcset(srcset.value, resolveOpts);
}
// we have an array of values, these srcsets came from extractFromLocalDoc
var len = srcset.values.length;
for (var i = 0; i < len; i++) {
var ssv = srcset.values[i];
resolveOpts.mod = ssv.mod;
resolveOpts.tagSrc = ssv.tagSrc;
this.extractSrcSetNotPreSplit(ssv.srcset, resolveOpts);
AutoFetcher.prototype.handleSrcset = function(srcset, context) {
var resolveOpts = { docBaseURI: context.docBaseURI };
if (srcset.value) {
// we have a single value, this srcset came from either
// preserveDataSrcset (not presplit) preserveSrcset (presplit)
resolveOpts.mod = srcset.mod;
if (!srcset.presplit) {
// extract URLs from the srcset string
return this.extractSrcSetNotPreSplit(srcset.value, resolveOpts);
}
// we have an array of srcset URL strings
return this.extractSrcset(srcset.value, resolveOpts);
}
// we have an array of values, these srcsets came from extractFromLocalDoc
var len = srcset.values.length;
for (var i = 0; i < len; i++) {
var ssv = srcset.values[i];
resolveOpts.mod = ssv.mod;
resolveOpts.tagSrc = ssv.tagSrc;
this.extractSrcSetNotPreSplit(ssv.srcset, resolveOpts);
}
};
AutoFetcher.prototype.autoFetch = function(data) {
// we got a message and now we autofetch!
// these calls turn into no ops if they have no work
if (data.media) {
this.handleMedia(data.media);
}
AutoFetcher.prototype.autoFetch = function (data) {
// we got a message and now we autofetch!
// these calls turn into no ops if they have no work
if (data.media) {
this.handleMedia(data.media);
}
if (data.src) {
this.handleSrc(data.src, data.context || {});
}
if (data.src) {
this.handleSrc(data.src, data.context || {});
}
if (data.srcset) {
this.handleSrcset(data.srcset, data.context || {});
}
if (data.srcset) {
this.handleSrcset(data.srcset, data.context || {});
}
this.fetchImgs();
this.fetchAV();
this.fetchImgs();
this.fetchAV();
};
// initialize ourselves from the query params :)
try {
var loc = new self.URL(location.href);
autofetcher = new AutoFetcher(JSON.parse(loc.searchParams.get('init')));
var loc = new self.URL(location.href);
autofetcher = new AutoFetcher(JSON.parse(loc.searchParams.get('init')));
} catch (e) {
// likely we are in an older version of safari
var search = decodeURIComponent(location.search.split('?')[1]).split('&');
var init = JSON.parse(search[0].substr(search[0].indexOf('=') + 1));
init.prefix = decodeURIComponent(init.prefix);
init.baseURI = decodeURIComponent(init.baseURI);
autofetcher = new AutoFetcher(init);
// likely we are in an older version of safari
var search = decodeURIComponent(location.search.split('?')[1]).split('&');
var init = JSON.parse(search[0].substr(search[0].indexOf('=') + 1));
init.prefix = decodeURIComponent(init.prefix);
init.baseURI = decodeURIComponent(init.baseURI);
autofetcher = new AutoFetcher(init);
}

468
pywb/static/autoFetchWorkerProxyMode.js
View File

@ -15,271 +15,289 @@ var autofetcher = null;
function noop() {}
if (typeof self.Promise === 'undefined') {
// not kewl we must polyfill Promise
self.Promise = function (executor) {
executor(noop, noop);
};
self.Promise.prototype.then = function (cb) {
if (cb) cb();
return this;
};
self.Promise.prototype.catch = function () {
return this;
};
self.Promise.all = function (values) {
return new Promise(noop);
};
// not kewl we must polyfill Promise
self.Promise = function(executor) {
executor(noop, noop);
};
self.Promise.prototype.then = function(cb) {
if (cb) cb();
return this;
};
self.Promise.prototype.catch = function() {
return this;
};
self.Promise.all = function(values) {
return new Promise(noop);
};
}
if (typeof self.fetch === 'undefined') {
// not kewl we must polyfill fetch.
self.fetch = function (url) {
return new Promise(function (resolve) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.send();
resolve();
});
};
// not kewl we must polyfill fetch.
self.fetch = function(url) {
return new Promise(function(resolve) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.send();
resolve();
});
};
}
self.onmessage = function (event) {
var data = event.data;
switch (data.type) {
case 'values':
autofetcher.autofetchMediaSrcset(data);
break;
case 'fetch-all':
autofetcher.justFetch(data);
break;
}
self.onmessage = function(event) {
var data = event.data;
switch (data.type) {
case 'values':
autofetcher.autofetchMediaSrcset(data);
break;
case 'fetch-all':
autofetcher.justFetch(data);
break;
}
};
function AutoFetcher() {
if (!(this instanceof AutoFetcher)) {
return new AutoFetcher();
}
// local cache of URLs fetched, to reduce server load
this.seen = {};
// array of URLs to be fetched
this.queue = [];
this.avQueue = [];
// should we queue a URL or not
this.queuing = false;
// a URL to resolve relative URLs found in the cssText of CSSMedia rules.
this.currentResolver = null;
// should we queue a URL or not
this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this);
this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
if (!(this instanceof AutoFetcher)) {
return new AutoFetcher();
}
// local cache of URLs fetched, to reduce server load
this.seen = {};
// array of URLs to be fetched
this.queue = [];
this.avQueue = [];
// should we queue a URL or not
this.queuing = false;
// a URL to resolve relative URLs found in the cssText of CSSMedia rules.
this.currentResolver = null;
// should we queue a URL or not
this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this);
this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
}
AutoFetcher.prototype.delay = function () {
return new Promise(function (resolve, reject) {
setTimeout(resolve, FetchDelay);
AutoFetcher.prototype.delay = function() {
return new Promise(function(resolve, reject) {
setTimeout(resolve, FetchDelay);
});
};
AutoFetcher.prototype.imgFetchDone = function() {
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function() {
autofetcher.queuing = false;
autofetcher.fetchImgs();
});
} else {
this.queuing = false;
}
};
AutoFetcher.prototype.imgFetchDone = function () {
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuing = false;
autofetcher.fetchImgs();
});
} else {
this.queuing = false;
}
AutoFetcher.prototype.avFetchDone = function() {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function() {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
}
};
AutoFetcher.prototype.avFetchDone = function () {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
AutoFetcher.prototype.fetchAV = function() {
if (this.queuingAV || this.avQueue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (
this.avQueue.length > 0 &&
runningFetchers.length <= DefaultNumAvFetches
) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop));
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop));
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
};
AutoFetcher.prototype.fetchAV = function () {
if (this.queuingAV || this.avQueue.length === 0) {
return;
AutoFetcher.prototype.fetchImgs = function() {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch
this.queuing = true;
var runningFetchers = [];
while (
this.queue.length > 0 &&
runningFetchers.length <= DefaultNumImFetches
) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop));
}
if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop));
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (this.avQueue.length > 0 && runningFetchers.length <= DefaultNumAvFetches) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
}
Promise.all(runningFetchers)
.then(this.imgFetchDone)
.catch(this.imgFetchDone);
};
AutoFetcher.prototype.fetchImgs = function () {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch
this.queuing = true;
var runningFetchers = [];
while (this.queue.length > 0 && runningFetchers.length <= DefaultNumImFetches) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.imgFetchDone)
.catch(this.imgFetchDone);
AutoFetcher.prototype.queueNonAVURL = function(url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.queue.push(url);
};
AutoFetcher.prototype.queueNonAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.queue.push(url);
AutoFetcher.prototype.queueAVURL = function(url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.queueAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.safeResolve = function (url, resolver) {
// Guard against the exception thrown by the URL constructor if the URL or resolver is bad
// if resolver is undefined/null then this function passes url through
var resolvedURL = url;
if (resolver) {
try {
resolvedURL = (new URL(url, resolver)).href
} catch (e) {
resolvedURL = url;
}
AutoFetcher.prototype.safeResolve = function(url, resolver) {
// Guard against the exception thrown by the URL constructor if the URL or resolver is bad
// if resolver is undefined/null then this function passes url through
var resolvedURL = url;
if (resolver) {
try {
resolvedURL = new URL(url, resolver).href;
} catch (e) {
resolvedURL = url;
}
return resolvedURL;
}
return resolvedURL;
};
AutoFetcher.prototype.urlExtractor = function (match, n1, n2, n3, offset, string) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
// this.currentResolver is set to the URL which the browser would normally
// resolve relative urls with (URL of the stylesheet) in an exceptionless manner
// (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(n2, this.currentResolver);
if (resolvedURL) {
this.queueNonAVURL(resolvedURL);
}
return n1 + n2 + n3;
AutoFetcher.prototype.urlExtractor = function(
match,
n1,
n2,
n3,
offset,
string
) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
// this.currentResolver is set to the URL which the browser would normally
// resolve relative urls with (URL of the stylesheet) in an exceptionless manner
// (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(n2, this.currentResolver);
if (resolvedURL) {
this.queueNonAVURL(resolvedURL);
}
return n1 + n2 + n3;
};
AutoFetcher.prototype.extractMedia = function (mediaRules) {
// this is a broken down rewrite_style
if (mediaRules == null) return;
for (var i = 0; i < mediaRules.length; i++) {
// set currentResolver to the value of this stylesheets URL, done to ensure we do not have to
// create functions on each loop iteration because we potentially create a new `URL` object
// twice per iteration
this.currentResolver = mediaRules[i].resolve;
mediaRules[i].cssText
.replace(STYLE_REGEX, this.urlExtractor)
.replace(IMPORT_REGEX, this.urlExtractor);
}
AutoFetcher.prototype.extractMedia = function(mediaRules) {
// this is a broken down rewrite_style
if (mediaRules == null) return;
for (var i = 0; i < mediaRules.length; i++) {
// set currentResolver to the value of this stylesheets URL, done to ensure we do not have to
// create functions on each loop iteration because we potentially create a new `URL` object
// twice per iteration
this.currentResolver = mediaRules[i].resolve;
mediaRules[i].cssText
.replace(STYLE_REGEX, this.urlExtractor)
.replace(IMPORT_REGEX, this.urlExtractor);
}
};
AutoFetcher.prototype.extractSrcset = function (srcsets) {
// preservation worker in proxy mode sends us the value of the srcset attribute of an element
// and a URL to correctly resolve relative URLS. Thus we must recreate rewrite_srcset logic here
if (srcsets == null) return;
var length = srcsets.length;
var extractedSrcSet, srcsetValue, ssSplit, j;
for (var i = 0; i < length; i++) {
extractedSrcSet = srcsets[i];
ssSplit = extractedSrcSet.srcset.split(srcsetSplit);
for (j = 0; j < ssSplit.length; j++) {
if (ssSplit[j]) {
srcsetValue = ssSplit[j].trim();
if (srcsetValue.length > 0) {
// resolve the URL in an exceptionless manner (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(srcsetValue.split(' ')[0], extractedSrcSet.resolve);
if (resolvedURL) {
if (extractedSrcSet.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
} else {
this.queueAVURL(resolvedURL);
}
}
}
}
}
}
};
AutoFetcher.prototype.extractSrc = function (srcVals) {
// preservation worker in proxy mode sends us the value of the srcset attribute of an element
// and a URL to correctly resolve relative URLS. Thus we must recreate rewrite_srcset logic here
if (srcVals == null || srcVals.length === 0) return;
var length = srcVals.length;
var srcVal;
for (var i = 0; i < length; i++) {
srcVal = srcVals[i];
var resolvedURL = this.safeResolve(srcVal.src, srcVal.resolve);
if (resolvedURL) {
if (srcVal.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
AutoFetcher.prototype.extractSrcset = function(srcsets) {
// preservation worker in proxy mode sends us the value of the srcset attribute of an element
// and a URL to correctly resolve relative URLS. Thus we must recreate rewrite_srcset logic here
if (srcsets == null) return;
var length = srcsets.length;
var extractedSrcSet, srcsetValue, ssSplit, j;
for (var i = 0; i < length; i++) {
extractedSrcSet = srcsets[i];
ssSplit = extractedSrcSet.srcset.split(srcsetSplit);
console.log(ssSplit);
for (j = 0; j < ssSplit.length; j++) {
if (ssSplit[j]) {
srcsetValue = ssSplit[j].trim();
if (srcsetValue.length > 0) {
// resolve the URL in an exceptionless manner (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(
srcsetValue.split(' ')[0],
extractedSrcSet.resolve
);
if (resolvedURL) {
if (extractedSrcSet.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
} else {
this.queueAVURL(resolvedURL);
this.queueAVURL(resolvedURL);
}
} else {
console.log(resolvedURL);
}
} else {
console.log(srcsetValue);
}
}
}
}
};
AutoFetcher.prototype.autofetchMediaSrcset = function (data) {
// we got a message and now we autofetch!
// these calls turn into no ops if they have no work
this.extractMedia(data.media);
this.extractSrcset(data.srcset);
this.extractSrc(data.src);
this.fetchImgs();
this.fetchAV();
AutoFetcher.prototype.extractSrc = function(srcVals) {
// preservation worker in proxy mode sends us the value of the srcset attribute of an element
// and a URL to correctly resolve relative URLS. Thus we must recreate rewrite_srcset logic here
if (srcVals == null || srcVals.length === 0) return;
var length = srcVals.length;
var srcVal;
for (var i = 0; i < length; i++) {
srcVal = srcVals[i];
var resolvedURL = this.safeResolve(srcVal.src, srcVal.resolve);
if (resolvedURL) {
if (srcVal.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
} else {
this.queueAVURL(resolvedURL);
}
}
}
};
AutoFetcher.prototype.justFetch = function (data) {
// we got a message containing only urls to be fetched
if (data == null || data.values == null) return;
for (var i = 0; i < data.values.length; ++i) {
this.queueNonAVURL(data.values[i]);
}
this.fetchImgs();
AutoFetcher.prototype.autofetchMediaSrcset = function(data) {
// we got a message and now we autofetch!
// these calls turn into no ops if they have no work
this.extractMedia(data.media);
this.extractSrcset(data.srcset);
this.extractSrc(data.src);
this.fetchImgs();
this.fetchAV();
};
AutoFetcher.prototype.justFetch = function(data) {
// we got a message containing only urls to be fetched
if (data == null || data.values == null) return;
for (var i = 0; i < data.values.length; ++i) {
this.queueNonAVURL(data.values[i]);
}
this.fetchImgs();
};
autofetcher = new AutoFetcher();

4251
pywb/static/wombat.js
View File

File diff suppressed because one or more lines are too long

463
pywb/static/wombatProxyMode.js
View File

File diff suppressed because one or more lines are too long

19
pywb/static/wombatWorkers.js Normal file
View File

File diff suppressed because one or more lines are too long

82
pywb/static/ww_rw.js
View File

@ -1,82 +0,0 @@
// pywb mini rewriter for injection into web worker scripts
function WBWombat(info) {
function maybeResolveURL(origURL) {
try {
var resolved = new URL(origURL, info.originalURL);
return resolved.href;
} catch (e) {
return origURL;
}
}
function rewrite_url(url) {
if (url.indexOf('blob:') === 0) return url;
if (url && info.originalURL && url.indexOf('/') === 0) {
url = maybeResolveURL(url);
}
if (info.prefix) {
return info.prefix + url;
}
return url;
}
function init_ajax_rewrite() {
var orig = self.XMLHttpRequest.prototype.open;
function open_rewritten(method, url, async, user, password) {
url = rewrite_url(url);
// defaults to true
if (async != false) {
async = true;
}
var result = orig.call(this, method, url, async, user, password);
if (url.indexOf('data:') !== 0) {
this.setRequestHeader('X-Pywb-Requested-With', 'XMLHttpRequest');
}
}
self.XMLHttpRequest.prototype.open = open_rewritten;
}
init_ajax_rewrite();
function rewriteArgs(argsObj) {
// recreate the original arguments object just with URLs rewritten
var newArgObj = new Array(argsObj.length);
for (var i = 0; i < newArgObj.length; i++) {
var arg = argsObj[i];
newArgObj[i] = rewrite_url(arg);
}
return newArgObj;
}
var origImportScripts = self.importScripts;
self.importScripts = function importScripts() {
// rewrite the arguments object and call original function via fn.apply
var rwArgs = rewriteArgs(arguments);
return origImportScripts.apply(this, rwArgs);
};
if (self.fetch != null) {
// this fetch is Worker.fetch
var orig_fetch = self.fetch;
self.fetch = function(input, init_opts) {
var inputType = typeof(input);
if (inputType === 'string') {
input = rewrite_url(input);
} else if (inputType === 'object' && input.url) {
var new_url = rewrite_url(input.url);
if (new_url !== input.url) {
input = new Request(new_url, input);
}
}
init_opts = init_opts || {};
init_opts['credentials'] = 'include';
return orig_fetch.call(this, input, init_opts);
};
}
}

2
pywb/version.py
View File

@ -1 +1 @@
__version__ = '2.2.20190410'
__version__ = '2.3.0.dev0'

16
tests/base_config_test.py
View File

@ -23,6 +23,9 @@ def fmod_sl(request):
# ============================================================================
class BaseConfigTest(BaseTestClass):
lint_app = True
extra_headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'
}
@classmethod
def get_test_app(cls, config_file, custom_config=None):
@ -62,21 +65,34 @@ class BaseConfigTest(BaseTestClass):
assert resp.content_length > 0
def get(self, url, fmod, *args, **kwargs):
self.__ensure_headers(kwargs)
app = self.testapp if fmod else self.testapp_non_frame
return app.get(url.format(fmod), *args, **kwargs)
def post(self, url, fmod, *args, **kwargs):
self.__ensure_headers(kwargs)
app = self.testapp if fmod else self.testapp_non_frame
return app.post(url.format(fmod), *args, **kwargs)
def post_json(self, url, fmod, *args, **kwargs):
self.__ensure_headers(kwargs)
app = self.testapp if fmod else self.testapp_non_frame
return app.post_json(url.format(fmod), *args, **kwargs)
def head(self, url, fmod, *args, **kwargs):
self.__ensure_headers(kwargs)
app = self.testapp if fmod else self.testapp_non_frame
return app.head(url.format(fmod), *args, **kwargs)
def __ensure_headers(self, kwargs):
if 'headers' in kwargs:
headers = kwargs.get('headers')
else:
headers = kwargs['headers'] = {}
if isinstance(headers, dict) and 'User-Agent' not in headers:
headers['User-Agent'] = self.extra_headers['User-Agent']
#=============================================================================
class CollsDirMixin(TempDirTests):

2
tests/test_root_coll.py
View File

@ -31,7 +31,7 @@ class TestRootColl(BaseConfigTest):
def test_root_replay_redir(self, fmod):
resp = self.get('/20140128051539{0}/http://www.iana.org/domains/example', fmod)
assert resp.status_int == 302
assert resp.status_int in (301, 302)
assert resp.headers['Location'] == 'http://localhost:80/20140128051539{0}/https://www.iana.org/domains/reserved'.format(fmod)

22
wombat/.eslintrc.json Normal file
View File

@ -0,0 +1,22 @@
{
"extends": ["plugin:prettier/recommended"],
"env": {
"browser": true,
"node": true
},
"rules": {
"camelcase": "off",
"no-fallthrough": "off"
},
"globals": {
"chai": true,
"mocha": true,
"WB_wombat_location": true,
"expect": true,
"_": true
},
"parserOptions": {
"ecmaVersion": 9,
"sourceType": "module"
}
}

22
wombat/boostrap.sh Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
SELF_D=$(dirname ${BASH_SOURCE[0]})
ASSETSPath="${SELF_D}/test/assets"
NODEM="${SELF_D}/node_modules"
INTERNAL="${SELF_D}/internal"
ROLLUP="${NODEM}/.bin/rollup"
if hash yarn 2>/dev/null; then
yarn install
else
npm install
fi
printf "\nBuilding wombat in prod mode"
node ${ROLLUP} -c rollup.config.prod.js
printf "\nBootstrapping tests"
cp "../pywb/static/css/bootstrap.min.css" "${ASSETSPath}/bootstrap.min.css"
node ${ROLLUP} -c "${INTERNAL}/rollup.testPageBundle.config.js"
node ${ROLLUP} -c "${SELF_D}/rollup.config.test.js"

30
wombat/internal/rollup.testPageBundle.config.js Normal file
View File

@ -0,0 +1,30 @@
import * as path from 'path';
import resolve from 'rollup-plugin-node-resolve';
const wombatDir = path.join(__dirname, '..');
const moduleDirectory = path.join(wombatDir, 'node_modules');
const baseTestOutput = path.join(wombatDir, 'test', 'assets');
const noStrict = {
renderChunk(code) {
return code.replace("'use strict';", '');
}
};
export default {
input: path.join(__dirname, 'testPageBundle.js'),
output: {
name: 'testPageBundle',
file: path.join(baseTestOutput, 'testPageBundle.js'),
sourcemap: false,
format: 'es'
},
plugins: [
resolve({
customResolveOptions: {
moduleDirectory
}
}),
noStrict
]
};

190
wombat/internal/testPageBundle.js Normal file
View File

@ -0,0 +1,190 @@
import get from 'lodash-es/get';
/**
* @type {TestOverwatch}
*/
window.TestOverwatch = class TestOverwatch {
/**
* @param {Object} domStructure
*/
constructor(domStructure) {
/**
* @type {{document: Document, window: Window}}
*/
this.ownContextWinDoc = { window, document };
this.wbMessages = { load: false };
this.domStructure = domStructure;
/**
* @type {HTMLIFrameElement}
*/
this.sandbox = domStructure.sandbox;
window.addEventListener(
'message',
event => {
if (event.data) {
const { data } = event;
switch (data.wb_type) {
case 'load':
this.wbMessages.load =
this.wbMessages.load || data.readyState === 'complete';
this.domStructure.load.url.data = data.url;
this.domStructure.load.title.data = data.title;
this.domStructure.load.readyState.data = data.readyState;
break;
case 'replace-url':
this.wbMessages['replace-url'] = data;
this.domStructure.replaceURL.url.data = data.url;
this.domStructure.replaceURL.title.data = data.title;
break;
case 'title':
this.wbMessages.title = data.title;
this.domStructure.titleMsg.data = data.title;
break;
case 'hashchange':
this.domStructure.hashchange.data = data.title;
this.wbMessages.hashchange = data.hash;
break;
case 'cookie':
this.domStructure.cookie.domain = data.domain;
this.domStructure.cookie.cookie = data.cookie;
this.wbMessages.cookie = data;
break;
default:
this.domStructure.unknown.data = JSON.stringify(data);
break;
}
}
},
false
);
}
/**
* This function initializes the wombat in the sandbox and ads a single
* additional property to the sandbox's window WombatTestUtil, an object that provides
* any functionality required for tests. This is done in order to ensure testing
* environment purity.
*/
initSandbox() {
this.domStructure.reset();
this.wbMessages = { load: false };
this.sandbox.contentWindow._WBWombatInit(this.sandbox.contentWindow.wbinfo);
this.sandbox.contentWindow.WombatTestUtil = {
didWombatSendTheLoadMsg: () => this.wbMessages.load,
wombatSentTitleUpdate: () => this.wbMessages.title,
wombatSentReplaceUrlMsg: () => this.wbMessages['replace-url'],
createUntamperedWithElement: init => this.createElement(init),
getElementPropertyAsIs: (elem, prop) =>
this.getElementPropertyAsIs(elem, prop),
getElementNSPropertyAsIs: (elem, ns, prop) =>
this.getElementNSPropertyAsIs(elem, ns, prop),
getOriginalWinDomViaPath: objectPath =>
this.getOriginalWinDomViaPath(objectPath),
getViaPath: (obj, objectPath) => get(obj, objectPath),
getOriginalPropertyDescriptorFor: (elem, prop) =>
this.ownContextWinDoc.window.Reflect.getOwnPropertyDescriptor(
elem,
prop
),
getStylePropertyAsIs: (elem, prop) =>
this.getStylePropertyAsIs(elem, prop),
getCSSPropertyAsIs: (elem, prop) => this.getCSSPropertyAsIs(elem, prop)
};
}
maybeInitSandbox() {
if (this.sandbox.contentWindow.WB_wombat_location != null) return;
this.initSandbox();
}
/**
* Creates a DOM element(s) based on the supplied creation options.
* @param {Object} init - Options for new element creation
* @return {HTMLElement|Node} - The newly created element
*/
createElement(init) {
if (typeof init === 'string') {
return this.ownContextWinDoc.document.createElement(init);
}
if (init.tag === 'textNode') {
var tn = this.ownContextWinDoc.document.createTextNode(init.value || '');
if (init.ref) init.ref(tn);
return tn;
}
var elem = this.ownContextWinDoc.document.createElement(init.tag);
if (init.id) elem.id = init.id;
if (init.className) elem.className = init.className;
if (init.style) elem.style = init.style;
if (init.innerText) elem.innerText = init.innerText;
if (init.innerHTML) elem.innerHTML = init.innerHTML;
if (init.attributes) {
var atts = init.attributes;
for (var attributeName in atts) {
elem.setAttribute(attributeName, atts[attributeName]);
}
}
if (init.dataset) {
var dataset = init.dataset;
for (var dataName in dataset) {
elem.dataset[dataName] = dataset[dataName];
}
}
if (init.events) {
var events = init.events;
for (var eventName in events) {
elem.addEventListener(eventName, events[eventName]);
}
}
if (init.child) {
elem.appendChild(this.createElement(init.child));
}
if (init.children) {
var kids = init.children;
for (var i = 0; i < kids.length; i++) {
elem.appendChild(this.createElement(kids[i]));
}
}
if (init.ref) init.ref(elem);
return elem;
}
/**
* Returns the value of an elements property bypassing wombat rewriting
* @param {Node} elem
* @param {string} prop
* @return {*}
*/
getElementPropertyAsIs(elem, prop) {
return this.ownContextWinDoc.window.Element.prototype.getAttribute.call(
elem,
prop
);
}
getElementNSPropertyAsIs(elem, ns, prop) {
return this.ownContextWinDoc.window.Element.prototype.getAttributeNS.call(
elem,
ns,
prop
);
}
getStylePropertyAsIs(elem, prop) {
var getter = this.ownContextWinDoc.window.CSSStyleDeclaration.prototype.__lookupGetter__(
prop
);
return getter.call(elem, prop);
}
getCSSPropertyAsIs(elem, prop) {
return this.ownContextWinDoc.window.CSSStyleDeclaration.prototype.getPropertyValue.call(
elem,
prop
);
}
getOriginalWinDomViaPath(objectPath) {
return get(this.ownContextWinDoc, objectPath);
}
};

65
wombat/package.json Executable file
View File

@ -0,0 +1,65 @@
{
"name": "wombat",
"version": "2.53.0",
"main": "index.js",
"license": "GPL-3.0",
"devDependencies": {
"@types/fs-extra": "^5.0.5",
"ava": "^1.4.1",
"chokidar": "^2.1.5",
"chrome-remote-interface-extra": "^1.0.0",
"eslint": "^5.16.0",
"eslint-config-prettier": "^4.2.0",
"eslint-plugin-prettier": "^3.0.1",
"fastify": "^2.3.0",
"fastify-favicon": "^2.0.0",
"fastify-graceful-shutdown": "^2.0.1",
"fastify-static": "^2.4.0",
"fs-extra": "^7.0.1",
"lodash-es": "^4.17.11",
"prettier": "^1.17.0",
"rollup": "^1.10.1",
"rollup-plugin-babel-minify": "^8.0.0",
"rollup-plugin-cleanup": "^3.1.1",
"rollup-plugin-commonjs": "^9.3.4",
"rollup-plugin-node-resolve": "^4.2.3",
"rollup-plugin-uglify": "^6.0.2",
"rollup-plugin-uglify-es": "^0.0.1",
"tls-keygen": "^3.7.0"
},
"scripts": {
"build-prod": "rollup -c rollup.config.prod.js",
"build-dev": "ALL=1 rollup -c rollup.config.dev.js",
"build-dev-watch": "rollup --watch -c rollup.config.dev.js",
"build-dev-watch-proxy": "PROXY=1 rollup --watch -c rollup.config.dev.js",
"build-test": "rollup -c rollup.config.test.js && rollup -c ./internal/rollup.testPageBundle.config.js",
"build-full-test": "rollup -c rollup.config.test.js",
"build-test-bundle": "rollup -c ./internal/rollup.testPageBundle.config.js",
"test": "ava --verbose"
},
"prettier": {
"singleQuote": true
},
"ava": {
"concurrency": 1,
"verbose": true,
"serial": true,
"files": [
"test/*.js",
"!test/helpers/*.js",
"!test/assests/*",
"!test/helpers/extractOrigFunky.js"
],
"sources": [
"!src/**/*"
]
},
"resolutions": {
"*/**/graceful-fs": "~4.1.15",
"*/**/jsonfile": "~5.0.0",
"*/**/universalify": "~0.1.2"
},
"dependencies": {
"chrome-launcher": "^0.10.5"
}
}

78
wombat/rollup.config.dev.js Executable file
View File

@ -0,0 +1,78 @@
import * as path from 'path';
const basePywbOutput = path.join(__dirname, '..', 'pywb', 'static');
const noStrict = {
renderChunk(code) {
return code.replace("'use strict';", '');
}
};
const wombat = {
input: 'src/wbWombat.js',
output: {
name: 'wombat',
file: path.join(basePywbOutput, 'wombat.js'),
sourcemap: false,
format: 'iife'
},
watch: {
exclude: 'node_modules/**',
chokidar: {
alwaysStat: true,
usePolling: true
}
},
plugins: [noStrict]
};
const wombatProxyMode = {
input: 'src/wbWombatProxyMode.js',
output: {
name: 'wombat',
file: path.join(basePywbOutput, 'wombatProxyMode.js'),
sourcemap: false,
format: 'iife'
},
watch: {
exclude: 'node_modules/**',
chokidar: {
alwaysStat: true,
usePolling: true
}
},
plugins: [noStrict]
};
const wombatWorker = {
input: 'src/wombatWorkers.js',
output: {
name: 'wombatWorkers',
file: path.join(basePywbOutput, 'wombatWorkers.js'),
format: 'es',
sourcemap: false,
exports: 'none'
},
watch: {
exclude: 'node_modules/**',
chokidar: {
alwaysStat: true,
usePolling: true
}
},
plugins: [noStrict]
};
let config;
if (process.env.ALL) {
config = [wombat, wombatProxyMode, wombatWorker];
} else if (process.env.PROXY) {
config = wombatProxyMode;
} else if (process.env.WORKER) {
config = wombatProxyMode;
} else {
config = wombat;
}
export default config;

78
wombat/rollup.config.prod.js Executable file
View File

@ -0,0 +1,78 @@
import * as path from 'path';
import minify from 'rollup-plugin-babel-minify';
const license = `/*
Copyright(c) 2013-2018 Rhizome and Ilya Kreymer. Released under the GNU General Public License.
This file is part of pywb, https://github.com/webrecorder/pywb
pywb is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pywb is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pywb. If not, see <http://www.gnu.org/licenses/>.
*/`;
const basePywbOutput = path.join(__dirname, '..', 'pywb', 'static');
const addLicenceNoStrict = {
renderChunk(code) {
return `${license}\n${code.replace("'use strict';", '')}`;
}
};
const minificationOpts = {
booleans: false,
builtIns: false,
comments: false,
deadcode: false,
flipComparisons: false,
infinity: false,
keepClassName: true,
keepFnName: true,
mangle: false,
removeUndefined: false,
simplifyComparisons: false,
sourceMap: false,
typeConstructors: false,
undefinedToVoid: false
};
export default [
{
input: 'src/wbWombat.js',
plugins: [minify(minificationOpts), addLicenceNoStrict],
output: {
name: 'wombat',
file: path.join(basePywbOutput, 'wombat.js'),
format: 'iife'
}
},
{
input: 'src/wbWombatProxyMode.js',
plugins: [minify(minificationOpts), addLicenceNoStrict],
output: {
name: 'wombatProxyMode',
file: path.join(basePywbOutput, 'wombatProxyMode.js'),
format: 'iife'
}
},
{
input: 'src/wombatWorkers.js',
plugins: [minify(minificationOpts), addLicenceNoStrict],
output: {
name: 'wombatWorkers',
file: path.join(basePywbOutput, 'wombatWorkers.js'),
format: 'es',
sourcemap: false,
exports: 'none'
}
}
];

42
wombat/rollup.config.test.js Executable file
View File

@ -0,0 +1,42 @@
import * as path from 'path';
const baseTestOutput = path.join(__dirname, 'test', 'assets');
const noStrict = {
renderChunk(code) {
return code.replace("'use strict';", '');
}
};
export default [
{
input: 'src/wbWombat.js',
output: {
name: 'wombat',
file: path.join(baseTestOutput, 'wombat.js'),
sourcemap: false,
format: 'iife'
},
plugins: [noStrict]
},
{
input: 'src/wbWombatProxyMode.js',
output: {
name: 'wombat',
file: path.join(baseTestOutput, 'wombatProxyMode.js'),
sourcemap: false,
format: 'iife'
},
plugins: [noStrict]
},
{
input: 'src/wombatWorkers.js',
output: {
name: 'wombat',
file: path.join(baseTestOutput, 'wombatWorkers.js'),
format: 'es',
sourcemap: false,
exports: 'none'
},
plugins: [noStrict]
}
];

296
wombat/src/autoFetchWorker.js Executable file
View File

@ -0,0 +1,296 @@
/* eslint-disable camelcase */
import { autobind } from './wombatUtils';
/**
* @param {Wombat} wombat
*/
export default function AutoFetchWorker(wombat) {
if (!(this instanceof AutoFetchWorker)) {
return new AutoFetchWorker(wombat);
}
// specifically target the elements we desire
this.elemSelector =
'img[srcset], img[data-srcset], img[data-src], video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], ' +
'picture > source[srcset], picture > source[data-srcset], picture > source[data-src], ' +
'video > source[srcset], video > source[data-srcset], video > source[data-src], ' +
'audio > source[srcset], audio > source[data-srcset], audio > source[data-src]';
this.isTop = wombat.$wbwindow === wombat.$wbwindow.__WB_replay_top;
/** @type {Wombat} */
this.wombat = wombat;
/** @type {Window} */
this.$wbwindow = wombat.$wbwindow;
/** @type {?Worker|Object} */
this.worker = null;
autobind(this);
this._initWorker();
}
/**
* Initializes the backing worker IFF the execution context we are in is
* the replay tops otherwise creates a dummy worker that simply bounces the
* message that would have been sent to the backing worker to replay top.
*
* If creation of the worker fails, likely due to the execution context we
* are currently in having an null origin, we fallback to dummy worker creation.
* @private
*/
AutoFetchWorker.prototype._initWorker = function() {
var wombat = this.wombat;
if (this.isTop) {
// we are top and can will own this worker
// setup URL for the kewl case
// Normal replay and preservation mode pworker setup, its all one origin so YAY!
var workerURL =
(wombat.wb_info.auto_fetch_worker_prefix ||
wombat.wb_info.static_prefix) +
'autoFetchWorker.js?init=' +
encodeURIComponent(
JSON.stringify({
mod: wombat.wb_info.mod,
prefix: wombat.wb_abs_prefix,
rwRe: wombat.wb_unrewrite_rx
})
);
try {
this.worker = new Worker(workerURL);
return;
} catch (e) {
// it is likely we are in some kind of horrid iframe setup
// and the execution context we are currently in has a null origin
console.error(
'Failed to create auto fetch worker\n',
e,
'\nFalling back to non top behavior'
);
}
}
// add only the portions of the worker interface we use since we are not top
// and if in proxy mode start check polling
this.worker = {
postMessage: function(msg) {
if (!msg.wb_type) {
msg = { wb_type: 'aaworker', msg: msg };
}
wombat.$wbwindow.__WB_replay_top.__orig_postMessage(msg, '*');
},
terminate: function() {}
};
};
/**
* Extracts the media rules from the supplied CSSStyleSheet object if any
* are present and returns an array of the media cssText
* @param {CSSStyleSheet} sheet
* @return {Array<string>}
*/
AutoFetchWorker.prototype.extractMediaRulesFromSheet = function(sheet) {
var rules;
var media = [];
try {
rules = sheet.cssRules || sheet.rules;
} catch (e) {
return media;
}
// loop through each rule of the stylesheet
for (var i = 0; i < rules.length; ++i) {
var rule = rules[i];
if (rule.type === CSSRule.MEDIA_RULE) {
// we are a media rule so get its text
media.push(rule.cssText);
}
}
return media;
};
/**
* Extracts the media rules from the supplied CSSStyleSheet object if any
* are present after a tick of the event loop sending the results of the
* extraction to the backing worker
* @param {CSSStyleSheet|StyleSheet} sheet
*/
AutoFetchWorker.prototype.deferredSheetExtraction = function(sheet) {
var afw = this;
// defer things until next time the Promise.resolve Qs are cleared
Promise.resolve().then(function() {
// loop through each rule of the stylesheet
var media = afw.extractMediaRulesFromSheet(sheet);
if (media.length > 0) {
// we have some media rules to preserve
afw.preserveMedia(media);
}
});
};
/**
* Terminates the backing worker. This is a no op when we are not
* operating in the execution context of replay top
*/
AutoFetchWorker.prototype.terminate = function() {
// terminate the worker, a no op when not replay top
this.worker.terminate();
};
/**
* Sends a message to backing worker. If deferred is true
* the message is sent after one tick of the event loop
* @param {Object} msg
* @param {boolean} [deferred]
*/
AutoFetchWorker.prototype.postMessage = function(msg, deferred) {
if (deferred) {
var afWorker = this;
Promise.resolve().then(function() {
afWorker.worker.postMessage(msg);
});
return;
}
this.worker.postMessage(msg);
};
/**
* Sends the supplied srcset value to the backing worker for preservation
* @param {string|Array<string>} srcset
* @param {string} [mod]
*/
AutoFetchWorker.prototype.preserveSrcset = function(srcset, mod) {
// send values from rewriteSrcset to the worker
this.postMessage(
{
type: 'values',
srcset: { values: srcset, mod: mod, presplit: true }
},
true
);
};
/**
* Send the value of the supplied elements data-srcset attribute to the
* backing worker for preservation
* @param {Node} elem
*/
AutoFetchWorker.prototype.preserveDataSrcset = function(elem) {
// send values from rewriteAttr srcset to the worker deferred
// to ensure the page viewer sees the images first
this.postMessage(
{
type: 'values',
srcset: {
value: elem.dataset.srcset,
mod: this.rwMod(elem),
presplit: false
}
},
true
);
};
/**
* Sends the supplied array of cssText from media rules to the backing worker
* @param {Array<string>} media
*/
AutoFetchWorker.prototype.preserveMedia = function(media) {
// send CSSMediaRule values to the worker
this.postMessage({ type: 'values', media: media }, true);
};
/**
* Extracts the value of the srcset property if it exists from the supplied
* element
* @param {Element} elem
* @return {?string}
*/
AutoFetchWorker.prototype.getSrcset = function(elem) {
if (this.wombat.wb_getAttribute) {
return this.wombat.wb_getAttribute.call(elem, 'srcset');
}
return elem.getAttribute('srcset');
};
/**
* Returns the correct rewrite modifier for the supplied element
* @param {Element} elem
* @return {string}
*/
AutoFetchWorker.prototype.rwMod = function(elem) {
switch (elem.tagName) {
case 'SOURCE':
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE') {
return 'im_';
}
return 'oe_';
case 'IMG':
return 'im_';
}
return 'oe_';
};
/**
* Extracts the media rules from stylesheets and the (data-)srcset URLs from
* image elements the current context's document contains
*/
AutoFetchWorker.prototype.extractFromLocalDoc = function() {
// get the values to be preserved from the documents stylesheets
// and all img, video, audio elements with (data-)?srcset or data-src
var afw = this;
Promise.resolve().then(function() {
var msg = {
type: 'values',
context: { docBaseURI: document.baseURI }
};
var media = [];
var i = 0;
var sheets = document.styleSheets;
for (; i < sheets.length; ++i) {
media = media.concat(afw.extractMediaRulesFromSheet(sheets[i]));
}
var elems = document.querySelectorAll(afw.elemSelector);
var srcset = { values: [], presplit: false };
var src = { values: [] };
var elem, srcv, mod;
for (i = 0; i < elems.length; ++i) {
elem = elems[i];
// we want the original src value in order to resolve URLs in the worker when needed
srcv = elem.src ? elem.src : null;
// a from value of 1 indicates images and a 2 indicates audio/video
mod = afw.rwMod(elem);
if (elem.srcset) {
srcset.values.push({
srcset: afw.getSrcset(elem),
mod: mod,
tagSrc: srcv
});
}
if (elem.dataset.srcset) {
srcset.values.push({
srcset: elem.dataset.srcset,
mod: mod,
tagSrc: srcv
});
}
if (elem.dataset.src) {
src.values.push({ src: elem.dataset.src, mod: mod });
}
if (elem.tagName === 'SOURCE' && srcv) {
src.values.push({ src: srcv, mod: mod });
}
}
if (media.length) {
msg.media = media;
}
if (srcset.values.length) {
msg.srcset = srcset;
}
if (src.values.length) {
msg.src = src;
}
if (msg.media || msg.srcset || msg.src) {
afw.postMessage(msg);
}
});
};

468
wombat/src/autoFetchWorkerProxyMode.js Executable file
View File

@ -0,0 +1,468 @@
import { autobind } from './wombatUtils';
/**
* Create a new instance of AutoFetchWorkerProxyMode
* @param {Wombat} wombat
* @param {boolean} isTop
*/
export default function AutoFetchWorkerProxyMode(wombat, isTop) {
if (!(this instanceof AutoFetchWorkerProxyMode)) {
return new AutoFetchWorkerProxyMode(wombat, isTop);
}
/**
* @type {Wombat}
*/
this.wombat = wombat;
/**
* @type {boolean}
*/
this.isTop = isTop;
/**
* @type {?MutationObserver}
*/
this.mutationObz = null;
// specifically target the elements we desire
this.elemSelector =
'img[srcset], img[data-srcset], img[data-src], video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], ' +
'picture > source[srcset], picture > source[data-srcset], picture > source[data-src], ' +
'video > source[srcset], video > source[data-srcset], video > source[data-src], ' +
'audio > source[srcset], audio > source[data-srcset], audio > source[data-src]';
this.mutationOpts = {
characterData: false,
characterDataOldValue: false,
attributes: true,
attributeOldValue: true,
subtree: true,
childList: true,
attributeFilter: ['src', 'srcset']
};
autobind(this);
this._init(true);
}
/**
* Initialize the auto fetch worker
* @private
*/
AutoFetchWorkerProxyMode.prototype._init = function(first) {
var afwpm = this;
if (document.readyState === 'complete') {
this.styleTag = document.createElement('style');
this.styleTag.id = '$wrStyleParser$';
document.head.appendChild(this.styleTag);
if (this.isTop) {
// Cannot directly load our worker from the proxy origin into the current origin
// however we fetch it from proxy origin and can blob it into the current origin :)
fetch(this.wombat.wbAutoFetchWorkerPrefix).then(function(res) {
res
.text()
.then(function(text) {
var blob = new Blob([text], { type: 'text/javascript' });
afwpm.worker = new afwpm.wombat.$wbwindow.Worker(
URL.createObjectURL(blob)
);
afwpm.startChecking();
})
.catch(error => {
console.error(
'Could not create the backing worker for AutoFetchWorkerProxyMode'
);
console.error(error);
});
});
} else {
// add only the portions of the worker interface we use since we are not top and if in proxy mode start check polling
this.worker = {
postMessage: function(msg) {
if (!msg.wb_type) {
msg = { wb_type: 'aaworker', msg: msg };
}
afwpm.wombat.$wbwindow.top.postMessage(msg, '*');
},
terminate: function() {}
};
this.startChecking();
}
return;
}
if (!first) return;
var i = setInterval(function() {
if (document.readyState === 'complete') {
afwpm._init();
clearInterval(i);
}
}, 1000);
};
/**
* Initializes the mutation observer
*/
AutoFetchWorkerProxyMode.prototype.startChecking = function() {
this.extractFromLocalDoc();
this.mutationObz = new MutationObserver(this.mutationCB);
this.mutationObz.observe(document, this.mutationOpts);
};
/**
* Terminate the worker, a no op when not replay top
*/
AutoFetchWorkerProxyMode.prototype.terminate = function() {
this.worker.terminate();
};
/**
* Sends the supplied array of URLs to the backing worker
* @param {Array<string>} urls
*/
AutoFetchWorkerProxyMode.prototype.justFetch = function(urls) {
this.worker.postMessage({ type: 'fetch-all', values: urls });
};
/**
* Sends the supplied msg to the backing worker
* @param {Object} msg
*/
AutoFetchWorkerProxyMode.prototype.postMessage = function(msg) {
this.worker.postMessage(msg);
};
/**
* Handles an style, link or text node that was mutated. If the text argument
* is true the parent node of the text node is used otherwise the element itself
* @param {*} elem
* @param {Object} accum
* @param {boolean} [text]
* @return {void}
*/
AutoFetchWorkerProxyMode.prototype.handleMutatedStyleElem = function(
elem,
accum,
text
) {
var baseURI = document.baseURI;
var checkNode;
if (text) {
if (!elem.parentNode || elem.parentNode.localName !== 'style') return;
checkNode = elem.parentNode;
} else {
checkNode = elem;
}
try {
var extractedMedia = this.extractMediaRules(checkNode.sheet, baseURI);
if (extractedMedia.length) {
accum.media = accum.media.concat(extractedMedia);
return;
}
} catch (e) {}
if (!text && checkNode.href) {
accum.deferred.push(this.fetchCSSAndExtract(checkNode.href));
}
};
/**
* Handles extracting the desired values from the mutated element
* @param {*} elem
* @param {Object} accum
*/
AutoFetchWorkerProxyMode.prototype.handleMutatedElem = function(elem, accum) {
var baseURI = document.baseURI;
if (elem.nodeType === Node.TEXT_NODE) {
return this.handleMutatedStyleElem(elem, accum, true);
}
switch (elem.localName) {
case 'img':
case 'video':
case 'audio':
case 'source':
return this.handleDomElement(elem, baseURI, accum);
case 'style':
case 'link':
return this.handleMutatedStyleElem(elem, accum);
}
return this.extractSrcSrcsetFrom(elem, baseURI, accum);
};
/**
* Callback used for the mutation observer observe function
* @param {Array<MutationRecord>} mutationList
* @param {MutationObserver} observer
*/
AutoFetchWorkerProxyMode.prototype.mutationCB = function(
mutationList,
observer
) {
var accum = { type: 'values', srcset: [], src: [], media: [], deferred: [] };
for (var i = 0; i < mutationList.length; i++) {
var mutation = mutationList[i];
var mutationTarget = mutation.target;
this.handleMutatedElem(mutationTarget, accum);
if (mutation.type === 'childList' && mutation.addedNodes.length) {
var addedLen = mutation.addedNodes.length;
for (var j = 0; j < addedLen; j++) {
this.handleMutatedElem(mutation.addedNodes[j], accum);
}
}
}
// send what we have extracted, if anything, to the worker for processing
if (accum.deferred.length) {
var deferred = accum.deferred;
accum.deferred = null;
Promise.all(deferred).then(this.handleDeferredSheetResults);
}
if (accum.srcset.length || accum.src.length || accum.media.length) {
console.log('msg', Date.now(), accum);
this.postMessage(accum);
}
};
/**
* Returns T/F indicating if the supplied stylesheet object is to be skipped
* @param {StyleSheet} sheet
* @return {boolean}
*/
AutoFetchWorkerProxyMode.prototype.shouldSkipSheet = function(sheet) {
// we skip extracting rules from sheets if they are from our parsing style or come from pywb
if (sheet.id === '$wrStyleParser$') return true;
return !!(
sheet.href && sheet.href.indexOf(this.wombat.wb_info.proxy_magic) !== -1
);
};
/**
* Returns null if the supplied value is not usable for resolving rel URLs
* otherwise returns the supplied value
* @param {?string} srcV
* @return {null|string}
*/
AutoFetchWorkerProxyMode.prototype.validateSrcV = function(srcV) {
if (!srcV || srcV.indexOf('data:') === 0 || srcV.indexOf('blob:') === 0) {
return null;
}
return srcV;
};
/**
* Because this JS in proxy mode operates as it would on the live web
* the rules of CORS apply and we cannot rely on URLs being rewritten correctly
* fetch the cross origin css file and then parse it using a style tag to get the rules
* @param {string} cssURL
* @return {Promise<Array>}
*/
AutoFetchWorkerProxyMode.prototype.fetchCSSAndExtract = function(cssURL) {
var url =
location.protocol +
'//' +
this.wombat.wb_info.proxy_magic +
'/proxy-fetch/' +
cssURL;
var afwpm = this;
return fetch(url)
.then(function(res) {
return res.text().then(function(text) {
afwpm.styleTag.textContent = text;
return afwpm.extractMediaRules(afwpm.styleTag.sheet, cssURL);
});
})
.catch(function(error) {
return [];
});
};
/**
* Extracts CSSMedia rules from the supplied style sheet object
* @param {CSSStyleSheet|StyleSheet} sheet
* @param {string} baseURI
* @return {Array<Object>}
*/
AutoFetchWorkerProxyMode.prototype.extractMediaRules = function(
sheet,
baseURI
) {
// We are in proxy mode and must include a URL to resolve relative URLs in media rules
var results = [];
if (!sheet) return results;
var rules;
try {
rules = sheet.cssRules || sheet.rules;
} catch (e) {
return results;
}
if (!rules || rules.length === 0) return results;
var len = rules.length;
var resolve = sheet.href || baseURI;
for (var i = 0; i < len; ++i) {
var rule = rules[i];
if (rule.type === CSSRule.MEDIA_RULE) {
results.push({ cssText: rule.cssText, resolve: resolve });
}
}
return results;
};
/**
* Returns the correct rewrite modifier for the supplied element
* @param {Element} elem
* @return {string}
*/
AutoFetchWorkerProxyMode.prototype.rwMod = function(elem) {
switch (elem.tagName) {
case 'SOURCE':
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE') {
return 'im_';
}
return 'oe_';
case 'IMG':
return 'im_';
}
return 'oe_';
};
/**
* Extracts the srcset, data-[srcset, src], and src attribute (IFF source tag)
* from the supplied element
* @param {Element} elem
* @param {string} baseURI
* @param {?Object} acum
*/
AutoFetchWorkerProxyMode.prototype.handleDomElement = function(
elem,
baseURI,
acum
) {
// we want the original src value in order to resolve URLs in the worker when needed
var srcv = this.validateSrcV(elem.src);
var resolve = srcv || baseURI;
// get the correct mod in order to inform the backing worker where the URL(s) are from
var mod = this.rwMod(elem);
if (elem.srcset) {
if (acum.srcset == null) acum = { srcset: [] };
acum.srcset.push({ srcset: elem.srcset, resolve: resolve, mod: mod });
}
if (elem.dataset && elem.dataset.srcset) {
if (acum.srcset == null) acum = { srcset: [] };
acum.srcset.push({
srcset: elem.dataset.srcset,
resolve: resolve,
mod: mod
});
}
if (elem.dataset && elem.dataset.src) {
if (acum.src == null) acum.src = [];
acum.src.push({ src: elem.dataset.src, resolve: resolve, mod: mod });
}
if (elem.tagName === 'SOURCE' && srcv) {
if (acum.src == null) acum.src = [];
acum.src.push({ src: srcv, resolve: baseURI, mod: mod });
}
};
/**
* Calls {@link handleDomElement} for each element returned from
* calling querySelector({@link elemSelector}) on the supplied element.
*
* If the acum argument is not supplied the results of the extraction
* are sent to the backing worker
* @param {*} fromElem
* @param {string} baseURI
* @param {Object} [acum]
*/
AutoFetchWorkerProxyMode.prototype.extractSrcSrcsetFrom = function(
fromElem,
baseURI,
acum
) {
if (!fromElem.querySelectorAll) return;
// retrieve the auto-fetched elements from the supplied dom node
var elems = fromElem.querySelectorAll(this.elemSelector);
var len = elems.length;
var msg = acum != null ? acum : { type: 'values', srcset: [], src: [] };
for (var i = 0; i < len; i++) {
this.handleDomElement(elems[i], baseURI, msg);
}
// send what we have extracted, if anything, to the worker for processing
if (acum == null && (msg.srcset.length || msg.src.length)) {
this.postMessage(msg);
}
};
/**
* Sends the extracted media values to the backing worker
* @param {Array<Array<string>>} results
*/
AutoFetchWorkerProxyMode.prototype.handleDeferredSheetResults = function(
results
) {
if (results.length === 0) return;
var len = results.length;
var media = [];
for (var i = 0; i < len; ++i) {
media = media.concat(results[i]);
}
if (media.length) {
this.postMessage({ type: 'values', media: media });
}
};
/**
* Extracts CSS media rules from the supplied documents styleSheets list.
* If a document is not supplied the document used defaults to the current
* contexts document object
* @param {?Document} [doc]
*/
AutoFetchWorkerProxyMode.prototype.checkStyleSheets = function(doc) {
var media = [];
var deferredMediaExtraction = [];
var styleSheets = (doc || document).styleSheets;
var sheetLen = styleSheets.length;
for (var i = 0; i < sheetLen; i++) {
var sheet = styleSheets[i];
// if the sheet belongs to our parser node we must skip it
if (!this.shouldSkipSheet(sheet)) {
try {
// if no error is thrown due to cross origin sheet the urls then just add
// the resolved URLS if any to the media urls array
if (sheet.cssRules || sheet.rules) {
var extracted = this.extractMediaRules(sheet, doc.baseURI);
if (extracted.length) {
media = media.concat(extracted);
}
} else if (sheet.href != null) {
// depending on the browser cross origin stylesheets will have their
// cssRules property null but href non-null
deferredMediaExtraction.push(this.fetchCSSAndExtract(sheet.href));
}
} catch (error) {
// the stylesheet is cross origin and we must re-fetch via PYWB to get the contents for checking
if (sheet.href != null) {
deferredMediaExtraction.push(this.fetchCSSAndExtract(sheet.href));
}
}
}
}
if (media.length) {
// send
this.postMessage({ type: 'values', media: media });
}
if (deferredMediaExtraction.length) {
// wait for all our deferred fetching and extraction of cross origin
// stylesheets to complete and then send those values, if any, to the worker
Promise.all(deferredMediaExtraction).then(this.handleDeferredSheetResults);
}
};
/**
* Performs extraction from the current contexts document
*/
AutoFetchWorkerProxyMode.prototype.extractFromLocalDoc = function() {
// check for data-[src,srcset] and auto-fetched elems with srcset first
this.extractSrcSrcsetFrom(
this.wombat.$wbwindow.document,
this.wombat.$wbwindow.document.baseURI
);
// we must use the window reference passed to us to access this origins stylesheets
this.checkStyleSheets(this.wombat.$wbwindow.document);
};

137
wombat/src/customStorage.js Executable file
View File

@ -0,0 +1,137 @@
import { addToStringTagToClass, ensureNumber } from './wombatUtils';
/**
* A re-implementation of the Storage interface.
* This re-implementation is required for replay in order to ensure
* that web pages that require local or session storage work as expected as
* there is sometimes a limit on the amount of storage space that can be used.
* This re-implementation ensures that limit is unlimited as it would be in
* the live-web.
* @param {Wombat} wombat
* @param {string} proxying
* @see https://developer.mozilla.org/en-US/docs/Web/API/Storage
* @see https://html.spec.whatwg.org/multipage/webstorage.html#the-storage-interface
*/
export default function Storage(wombat, proxying) {
// hide our values from enumeration, spreed, et al
Object.defineProperties(this, {
data: {
enumerable: false,
value: {}
},
wombat: {
enumerable: false,
value: wombat
},
proxying: {
enumerable: false,
value: proxying
},
_deleteItem: {
enumerable: false,
value: function(item) {
delete this.data[item];
}
}
});
}
/**
* When passed a key name, will return that key's value
* @param {string} name
* @return {*}
*/
Storage.prototype.getItem = function getItem(name) {
return this.data.hasOwnProperty(name) ? this.data[name] : null;
};
/**
* When passed a key name and value, will add that key to the storage,
* or update that key's value if it already exists
* @param {string} name
* @param {*} value
* @return {*}
*/
Storage.prototype.setItem = function setItem(name, value) {
var sname = String(name);
var svalue = String(value);
var old = this.getItem(sname);
this.data[sname] = value;
this.fireEvent(sname, old, svalue);
return undefined;
};
/**
* When passed a key name, will remove that key from the storage
* @param {string} name
* @return {undefined}
*/
Storage.prototype.removeItem = function removeItem(name) {
var old = this.getItem(name);
this._deleteItem(name);
this.fireEvent(name, old, null);
return undefined;
};
/**
* When invoked, will empty all keys out of the storage
* @return {undefined}
*/
Storage.prototype.clear = function clear() {
this.data = {};
this.fireEvent(null, null, null);
return undefined;
};
/**
* When passed a number n, this method will return the name of the nth key in the storage
* @param {number} index
* @return {*}
*/
Storage.prototype.key = function key(index) {
var n = ensureNumber(index);
if (n == null || n < 0) return null;
var keys = Object.keys(this.data);
if (n < keys.length) return keys[n];
return null;
};
/**
* Because we are re-implementing the storage interface we must fire StorageEvent
* ourselves, this function does just that.
* @param {?string} key
* @param {*} oldValue
* @param {*} newValue
* @see https://html.spec.whatwg.org/multipage/webstorage.html#send-a-storage-notification
*/
Storage.prototype.fireEvent = function fireEvent(key, oldValue, newValue) {
var sevent = new StorageEvent('storage', {
key: key,
newValue: newValue,
oldValue: oldValue,
url: this.wombat.$wbwindow.WB_wombat_location.href
});
sevent._storageArea = this;
this.wombat.storage_listeners.map(sevent);
};
/**
* An override of the valueOf function that returns wombat's Proxy for the
* specific storage this class is for, either local or session storage.
* @return {Proxy<Storage>}
*/
Storage.prototype.valueOf = function valueOf() {
return this.wombat.$wbwindow[this.proxying];
};
// the length getter is on the prototype (__proto__ modern browsers)
Object.defineProperty(Storage.prototype, 'length', {
enumerable: false,
get: function length() {
return Object.keys(this.data).length;
}
});
addToStringTagToClass(Storage, 'Storage');

93
wombat/src/funcMap.js Executable file
View File

@ -0,0 +1,93 @@
/**
* A class that manages event listeners for the override applied to
* EventTarget.[addEventListener, removeEventListener]
*/
export default function FuncMap() {
/**
* @type {Array<Function[]>}
* @private
*/
this._map = [];
}
/**
* Adds a mapping of original listener -> wrapped original listener
* @param {Function} fnKey - The original listener function
* @param {Function} fnValue - The wrapped original listener function
*/
FuncMap.prototype.set = function(fnKey, fnValue) {
this._map.push([fnKey, fnValue]);
};
/**
* Returns the wrapped original listener that is mapped to the supplied function
* if it exists in the FuncMap's mapping
* @param {Function} fnKey - The original listener function
* @return {?Function}
*/
FuncMap.prototype.get = function(fnKey) {
for (var i = 0; i < this._map.length; i++) {
if (this._map[i][0] === fnKey) {
return this._map[i][1];
}
}
return null;
};
/**
* Returns the index of the wrapper for the supplied original function
* if it exists in the FuncMap's mapping
* @param {Function} fnKey - The original listener function
* @return {number}
*/
FuncMap.prototype.find = function(fnKey) {
for (var i = 0; i < this._map.length; i++) {
if (this._map[i][0] === fnKey) {
return i;
}
}
return -1;
};
/**
* Returns the wrapped original listener function for the supplied original
* listener function. If the wrapped original listener does not exist in
* FuncMap's mapping it is added.
* @param {Function} func - The original listener function
* @param {Function} initter - The a function that returns a wrapped version
* of the original listener function
* @return {?Function}
*/
FuncMap.prototype.add_or_get = function(func, initter) {
var fnValue = this.get(func);
if (!fnValue) {
fnValue = initter();
this.set(func, fnValue);
}
return fnValue;
};
/**
* Removes the mapping of the original listener function to its wrapped counter part
* @param {Function} func - The original listener function
* @return {?Function}
*/
FuncMap.prototype.remove = function(func) {
var idx = this.find(func);
if (idx >= 0) {
var fnMapping = this._map.splice(idx, 1);
return fnMapping[0][1];
}
return null;
};
/**
* Calls all wrapped listener functions contained in the FuncMap's mapping
* with the supplied param
* @param {*} param
*/
FuncMap.prototype.map = function(param) {
for (var i = 0; i < this._map.length; i++) {
this._map[i][1](param);
}
};

76
wombat/src/listeners.js Executable file
View File

@ -0,0 +1,76 @@
/* eslint-disable camelcase */
/**
*
* @param {Function} origListener
* @param {Window} win
* @return {Function}
*/
export function wrapSameOriginEventListener(origListener, win) {
return function wrappedSameOriginEventListener(event) {
if (window != win) {
return;
}
return origListener(event);
};
}
/**
* @param {Function} origListener
* @param {Object} obj
* @param {Wombat} wombat
* @return {Function}
*/
export function wrapEventListener(origListener, obj, wombat) {
return function wrappedEventListener(event) {
var ne;
if (event.data && event.data.from && event.data.message) {
if (
event.data.to_origin !== '*' &&
obj.WB_wombat_location &&
!wombat.startsWith(event.data.to_origin, obj.WB_wombat_location.origin)
) {
console.warn(
'Skipping message event to ' +
event.data.to_origin +
" doesn't start with origin " +
obj.WB_wombat_location.origin
);
return;
}
var source = event.source;
if (event.data.from_top) {
source = obj.__WB_top_frame;
} else if (
event.data.src_id &&
obj.__WB_win_id &&
obj.__WB_win_id[event.data.src_id]
) {
source = obj.__WB_win_id[event.data.src_id];
}
ne = new MessageEvent('message', {
bubbles: event.bubbles,
cancelable: event.cancelable,
data: event.data.message,
origin: event.data.from,
lastEventId: event.lastEventId,
source: wombat.proxyToObj(source),
ports: event.ports
});
ne._target = event.target;
ne._srcElement = event.srcElement;
ne._currentTarget = event.currentTarget;
ne._eventPhase = event.eventPhase;
ne._path = event.path;
} else {
ne = event;
}
return origListener(ne);
};
}

13
wombat/src/wbWombat.js Executable file
View File

@ -0,0 +1,13 @@
import Wombat from './wombat';
window._WBWombat = Wombat;
window._WBWombatInit = function(wbinfo) {
if (!this._wb_wombat || !this._wb_wombat.actual) {
var wombat = new Wombat(this, wbinfo);
wombat.actual = true;
this._wb_wombat = wombat.wombatInit();
this._wb_wombat.actual = true;
} else if (!this._wb_wombat) {
console.warn('_wb_wombat missing!');
}
};

13
wombat/src/wbWombatProxyMode.js Executable file
View File

@ -0,0 +1,13 @@
import WombatLite from './wombatLite';
window._WBWombat = WombatLite;
window._WBWombatInit = function(wbinfo) {
if (!this._wb_wombat || !this._wb_wombat.actual) {
var wombat = new WombatLite(this, wbinfo);
wombat.actual = true;
this._wb_wombat = wombat.wombatInit();
this._wb_wombat.actual = true;
} else if (!this._wb_wombat) {
console.warn('_wb_wombat missing!');
}
};

5664
wombat/src/wombat.js Executable file
View File

File diff suppressed because it is too large Load Diff

253
wombat/src/wombatLite.js Executable file
View File

@ -0,0 +1,253 @@
/* eslint-disable camelcase */
import AutoFetchWorkerProxyMode from './autoFetchWorkerProxyMode';
/**
* Wombat lite for proxy-mode
* @param {Window} $wbwindow
* @param {Object} wbinfo
*/
export default function WombatLite($wbwindow, wbinfo) {
if (!(this instanceof WombatLite)) return new WombatLite($wbwindow, wbinfo);
this.wb_info = wbinfo;
this.$wbwindow = $wbwindow;
this.wb_info.top_host = this.wb_info.top_host || '*';
this.wb_info.wombat_opts = this.wb_info.wombat_opts || {};
this.wbAutoFetchWorkerPrefix =
(this.wb_info.auto_fetch_worker_prefix || this.wb_info.static_prefix) +
'autoFetchWorkerProxyMode.js';
this.WBAutoFetchWorker = null;
}
/**
* Applies an override to Math.seed and Math.random using the supplied
* seed in order to ensure that random numbers are deterministic during
* replay
* @param {string} seed
*/
WombatLite.prototype.initSeededRandom = function(seed) {
// Adapted from:
// http://indiegamr.com/generate-repeatable-random-numbers-in-js/
this.$wbwindow.Math.seed = parseInt(seed);
var wombat = this;
this.$wbwindow.Math.random = function random() {
wombat.$wbwindow.Math.seed =
(wombat.$wbwindow.Math.seed * 9301 + 49297) % 233280;
return wombat.$wbwindow.Math.seed / 233280;
};
};
/**
* Applies an override to crypto.getRandomValues in order to make
* the values it returns are deterministic during replay
*/
WombatLite.prototype.initCryptoRandom = function() {
if (!this.$wbwindow.crypto || !this.$wbwindow.Crypto) return;
// var orig_getrandom = this.$wbwindow.Crypto.prototype.getRandomValues
var wombat = this;
var new_getrandom = function getRandomValues(array) {
for (var i = 0; i < array.length; i++) {
array[i] = parseInt(wombat.$wbwindow.Math.random() * 4294967296);
}
return array;
};
this.$wbwindow.Crypto.prototype.getRandomValues = new_getrandom;
this.$wbwindow.crypto.getRandomValues = new_getrandom;
};
/**
* Forces, when possible, the devicePixelRatio property of window to 1
* in order to ensure deterministic replay
*/
WombatLite.prototype.initFixedRatio = function() {
try {
// otherwise, just set it
this.$wbwindow.devicePixelRatio = 1;
} catch (e) {}
// prevent changing, if possible
if (Object.defineProperty) {
try {
// fixed pix ratio
Object.defineProperty(this.$wbwindow, 'devicePixelRatio', {
value: 1,
writable: false
});
} catch (e) {}
}
};
/**
* Applies an override to the Date object in order to ensure that
* all Dates used during replay are in the datetime of replay
* @param {string} timestamp
*/
WombatLite.prototype.initDateOverride = function(timestamp) {
if (this.$wbwindow.__wb_Date_now) return;
var newTimestamp = parseInt(timestamp) * 1000;
// var timezone = new Date().getTimezoneOffset() * 60 * 1000;
// Already UTC!
var timezone = 0;
var start_now = this.$wbwindow.Date.now();
var timediff = start_now - (newTimestamp - timezone);
var orig_date = this.$wbwindow.Date;
var orig_utc = this.$wbwindow.Date.UTC;
var orig_parse = this.$wbwindow.Date.parse;
var orig_now = this.$wbwindow.Date.now;
this.$wbwindow.__wb_Date_now = orig_now;
this.$wbwindow.Date = (function(Date_) {
return function Date(A, B, C, D, E, F, G) {
// Apply doesn't work for constructors and Date doesn't
// seem to like undefined args, so must explicitly
// call constructor for each possible args 0..7
if (A === undefined) {
return new Date_(orig_now() - timediff);
} else if (B === undefined) {
return new Date_(A);
} else if (C === undefined) {
return new Date_(A, B);
} else if (D === undefined) {
return new Date_(A, B, C);
} else if (E === undefined) {
return new Date_(A, B, C, D);
} else if (F === undefined) {
return new Date_(A, B, C, D, E);
} else if (G === undefined) {
return new Date_(A, B, C, D, E, F);
} else {
return new Date_(A, B, C, D, E, F, G);
}
};
})(this.$wbwindow.Date);
this.$wbwindow.Date.prototype = orig_date.prototype;
this.$wbwindow.Date.now = function now() {
return orig_now() - timediff;
};
this.$wbwindow.Date.UTC = orig_utc;
this.$wbwindow.Date.parse = orig_parse;
this.$wbwindow.Date.__WB_timediff = timediff;
Object.defineProperty(this.$wbwindow.Date.prototype, 'constructor', {
value: this.$wbwindow.Date
});
};
/**
* Applies an override that disables the pages ability to send OS native
* notifications. Also disables the ability of the replayed page to retrieve the geolocation
* of the view.
*
* This is done in order to ensure that no malicious abuse of these functions
* can happen during replay.
*/
WombatLite.prototype.initDisableNotifications = function() {
if (window.Notification) {
window.Notification.requestPermission = function requestPermission(
callback
) {
if (callback) {
// eslint-disable-next-line standard/no-callback-literal
callback('denied');
}
return Promise.resolve('denied');
};
}
var applyOverride = function(on) {
if (!on) return;
if (on.getCurrentPosition) {
on.getCurrentPosition = function getCurrentPosition(
success,
error,
options
) {
if (error) {
error({ code: 2, message: 'not available' });
}
};
}
if (on.watchPosition) {
on.watchPosition = function watchPosition(success, error, options) {
if (error) {
error({ code: 2, message: 'not available' });
}
};
}
};
if (window.geolocation) {
applyOverride(window.geolocation);
}
if (window.navigator.geolocation) {
applyOverride(window.navigator.geolocation);
}
};
/**
* Initializes and starts the auto-fetch worker IFF wbUseAFWorker is true
*/
WombatLite.prototype.initAutoFetchWorker = function() {
if (!this.$wbwindow.Worker) {
return;
}
var isTop = this.$wbwindow.self === this.$wbwindow.top;
if (this.$wbwindow.$WBAutoFetchWorker$ == null) {
this.WBAutoFetchWorker = new AutoFetchWorkerProxyMode(this, isTop);
// expose the WBAutoFetchWorker
Object.defineProperty(this.$wbwindow, '$WBAutoFetchWorker$', {
enumerable: false,
value: this.WBAutoFetchWorker
});
} else {
this.WBAutoFetchWorker = this.$wbwindow.$WBAutoFetchWorker$;
}
if (isTop) {
var wombatLite = this;
this.$wbwindow.addEventListener(
'message',
function(event) {
if (event.data && event.data.wb_type === 'aaworker') {
wombatLite.WBAutoFetchWorker.postMessage(event.data.msg);
}
},
false
);
}
};
/**
* Initialize wombat's internal state and apply all overrides
* @return {Object}
*/
WombatLite.prototype.wombatInit = function() {
if (this.wb_info.enable_auto_fetch && this.wb_info.is_live) {
this.initAutoFetchWorker();
}
// proxy mode overrides
// Random
this.initSeededRandom(this.wb_info.wombat_sec);
// Crypto Random
this.initCryptoRandom();
// set fixed pixel ratio
this.initFixedRatio();
// Date
this.initDateOverride(this.wb_info.wombat_sec);
// disable notifications
this.initDisableNotifications();
return { actual: false };
};

107
wombat/src/wombatLocation.js Executable file
View File

@ -0,0 +1,107 @@
/* eslint-disable camelcase */
import { addToStringTagToClass } from './wombatUtils';
/**
* A re-implementation of the Location interface that ensure that operations
* on the location interface behaves as expected during replay.
* @param {Location} orig_loc
* @param {Wombat} wombat
* @see https://developer.mozilla.org/en-US/docs/Web/API/Location
* @see https://html.spec.whatwg.org/multipage/browsers.html#the-location-interface
*/
export default function WombatLocation(orig_loc, wombat) {
// hide our values from enumeration, spreed, et al
Object.defineProperties(this, {
_orig_loc: {
configurable: true,
enumerable: false,
value: orig_loc
},
wombat: {
configurable: true,
enumerable: false,
value: wombat
},
orig_getter: {
enumerable: false,
value: function(prop) {
return this._orig_loc[prop];
}
},
orig_setter: {
enumerable: false,
value: function(prop, value) {
this._orig_loc[prop] = value;
}
}
});
wombat.initLocOverride(this, this.orig_setter, this.orig_getter);
wombat.setLoc(this, orig_loc.href);
for (var prop in orig_loc) {
if (!this.hasOwnProperty(prop) && typeof orig_loc[prop] !== 'function') {
this[prop] = orig_loc[prop];
}
}
}
/**
* Replaces the current resource with the one at the provided URL.
* The difference from the assign() method is that after using replace() the
* current page will not be saved in session History, meaning the user won't
* be able to use the back button to navigate to it.
* @param {string} url
* @return {*}
*/
WombatLocation.prototype.replace = function replace(url) {
var new_url = this.wombat.rewriteUrl(url);
var orig = this.wombat.extractOriginalURL(new_url);
if (orig === this.href) {
return orig;
}
return this._orig_loc.replace(new_url);
};
/**
* Loads the resource at the URL provided in parameter
* @param {string} url
* @return {*}
*/
WombatLocation.prototype.assign = function assign(url) {
var new_url = this.wombat.rewriteUrl(url);
var orig = this.wombat.extractOriginalURL(new_url);
if (orig === this.href) {
return orig;
}
return this._orig_loc.assign(new_url);
};
/**
* Reloads the resource from the current URL. Its optional unique parameter
* is a Boolean, which, when it is true, causes the page to always be reloaded
* from the server. If it is false or not specified, the browser may reload
* the page from its cache.
* @param {boolean} [forcedReload = false]
* @return {*}
*/
WombatLocation.prototype.reload = function reload(forcedReload) {
return this._orig_loc.reload(forcedReload || false);
};
/**
* @return {string}
*/
WombatLocation.prototype.toString = function toString() {
return this.href;
};
/**
* @return {WombatLocation}
*/
WombatLocation.prototype.valueOf = function valueOf() {
return this;
};
addToStringTagToClass(WombatLocation, 'Location');

56
wombat/src/wombatUtils.js Normal file
View File

@ -0,0 +1,56 @@
/**
* Ensures the supplied argument is a number or if it is not (can not be coerced to a number)
* this function returns null.
* @param {*} maybeNumber
* @return {?number}
*/
export function ensureNumber(maybeNumber) {
try {
switch (typeof maybeNumber) {
case 'number':
case 'bigint':
return maybeNumber;
}
var converted = Number(maybeNumber);
return !isNaN(converted) ? converted : null;
} catch (e) {}
return null;
}
/**
* Sets the supplied object's toStringTag IFF
* self.Symbol && self.Symbol.toStringTag are defined
* @param {Object} clazz
* @param {string} tag
*/
export function addToStringTagToClass(clazz, tag) {
if (
typeof self.Symbol !== 'undefined' &&
typeof self.Symbol.toStringTag !== 'undefined'
) {
Object.defineProperty(clazz.prototype, self.Symbol.toStringTag, {
value: tag,
enumerable: false
});
}
}
/**
* Binds every function this, except the constructor, of the supplied object
* to the instance of the supplied object
* @param {Object} clazz
*/
export function autobind(clazz) {
var proto = clazz.__proto__ || clazz.constructor.prototype || clazz.prototype;
var clazzProps = Object.getOwnPropertyNames(proto);
var len = clazzProps.length;
var prop;
var propValue;
for (var i = 0; i < len; i++) {
prop = clazzProps[i];
propValue = clazz[prop];
if (prop !== 'constructor' && typeof propValue === 'function') {
clazz[prop] = propValue.bind(clazz);
}
}
}

443
wombat/src/wombatWorkers.js Normal file
View File

@ -0,0 +1,443 @@
/**
* Mini wombat for performing URL rewriting within the
* Web/Shared/Service Worker context
* @param {Object} info
* @return {WBWombat}
*/
function WBWombat(info) {
if (!(this instanceof WBWombat)) return new WBWombat(info);
/** @type {Object} */
this.info = info;
this.initImportScriptsRewrite();
this.initHTTPOverrides();
this.initClientApisOverride();
this.initCacheApisOverride();
}
/**
* Returns T/F indicating if the supplied URL is not to be rewritten
* @param {string} url
* @return {boolean}
*/
WBWombat.prototype.noRewrite = function(url) {
return (
!url ||
url.indexOf('blob:') === 0 ||
url.indexOf('javascript:') === 0 ||
url.indexOf(this.info.prefix) === 0
);
};
/**
* Returns T/F indicating if the supplied URL is an relative URL
* @param {string} url
* @return {boolean}
*/
WBWombat.prototype.isRelURL = function(url) {
return url.indexOf('/') === 0 || url.indexOf('http:') !== 0;
};
/**
* Attempts to resolve the supplied relative URL against
* the origin this worker was created on
* @param {string} maybeRelURL
* @param {string} against
* @return {string}
*/
WBWombat.prototype.maybeResolveURL = function(maybeRelURL, against) {
if (!against) return maybeRelURL;
try {
var resolved = new URL(maybeRelURL, against);
return resolved.href;
} catch (e) {}
return maybeRelURL;
};
/**
* Returns null to indicate that the supplied URL is not to be rewritten.
* Otherwise returns a URL that can be rewritten
* @param {*} url
* @param {string} resolveAgainst
* @return {?string}
*/
WBWombat.prototype.ensureURL = function(url, resolveAgainst) {
if (!url) return url;
var newURL;
switch (typeof url) {
case 'string':
newURL = url;
break;
case 'object':
newURL = url.toString();
break;
default:
return null;
}
if (this.noRewrite(newURL)) return null;
if (this.isRelURL(newURL)) {
return this.maybeResolveURL(newURL, resolveAgainst);
}
return newURL;
};
/**
* Rewrites the supplied URL
* @param {string} url
* @return {string}
*/
WBWombat.prototype.rewriteURL = function(url) {
var rwURL = this.ensureURL(url, this.info.originalURL);
if (!rwURL) return url;
if (this.info.prefixMod) {
return this.info.prefixMod + rwURL;
}
return rwURL;
};
/**
* Rewrites the supplied URL of an controlled page using the mp\_ modifier
* @param {string} url
* @param {WindowClient} [client]
* @return {string}
*/
WBWombat.prototype.rewriteClientWindowURL = function(url, client) {
var rwURL = this.ensureURL(url, client ? client.url : this.info.originalURL);
if (!rwURL) return url;
if (this.info.prefix) {
return this.info.prefix + 'mp_/' + rwURL;
}
return rwURL;
};
/**
* Mini url rewriter specifically for rewriting web sockets
* @param {?string} originalURL
* @return {string}
*/
WBWombat.prototype.rewriteWSURL = function(originalURL) {
// If undefined, just return it
if (!originalURL) return originalURL;
var urltype_ = typeof originalURL;
var url = originalURL;
// If object, use toString
if (urltype_ === 'object') {
url = originalURL.toString();
} else if (urltype_ !== 'string') {
return originalURL;
}
// empty string check
if (!url) return url;
var wsScheme = 'ws://';
var wssScheme = 'wss://';
var https = 'https://';
var wbSecure = this.info.prefix.indexOf(https) === 0;
var wbPrefix =
this.info.prefix.replace(
wbSecure ? https : 'http://',
wbSecure ? wssScheme : wsScheme
) + 'ws_/';
return wbPrefix + url;
};
/**
* Rewrites all URLs in the supplied arguments object
* @param {Object} argsObj
* @return {Array<string>}
*/
WBWombat.prototype.rewriteArgs = function(argsObj) {
// recreate the original arguments object just with URLs rewritten
var newArgObj = new Array(argsObj.length);
for (var i = 0; i < newArgObj.length; i++) {
newArgObj[i] = this.rewriteURL(argsObj[i]);
}
return newArgObj;
};
/**
* Rewrites the input to one of the Fetch APIs
* @param {*|string|Request} input
* @return {*|string|Request}
*/
WBWombat.prototype.rewriteFetchApi = function(input) {
var rwInput = input;
switch (typeof input) {
case 'string':
rwInput = this.rewriteURL(input);
break;
case 'object':
if (input.url) {
var new_url = this.rewriteURL(input.url);
if (new_url !== input.url) {
// not much we can do here Request.url is read only
// https://developer.mozilla.org/en-US/docs/Web/API/Request/url
rwInput = new Request(new_url, input);
}
} else if (input.href) {
// it is likely that input is either self.location or self.URL
// we cant do anything here so just let it go
rwInput = input.href;
}
break;
}
return rwInput;
};
/**
* Rewrites the input to one of the Cache APIs
* @param {*|string|Request} request
* @return {*|string|Request}
*/
WBWombat.prototype.rewriteCacheApi = function(request) {
var rwRequest = request;
if (typeof request === 'string') {
rwRequest = this.rewriteURL(request);
}
return rwRequest;
};
/**
* Applies an override to the importScripts function
* @see https://html.spec.whatwg.org/multipage/workers.html#dom-workerglobalscope-importscripts
*/
WBWombat.prototype.initImportScriptsRewrite = function() {
if (!self.importScripts) return;
var wombat = this;
var origImportScripts = self.importScripts;
self.importScripts = function importScripts() {
// rewrite the arguments object and call original function via fn.apply
var rwArgs = wombat.rewriteArgs(arguments);
return origImportScripts.apply(this, rwArgs);
};
};
/**
* Applies overrides to the XMLHttpRequest.open and XMLHttpRequest.responseURL
* in order to ensure URLs are rewritten.
*
* Applies an override to window.fetch in order to rewrite URLs and URLs of
* the supplied Request objects used as arguments to fetch.
*
* Applies overrides to window.Request, window.Response, window.EventSource,
* and window.WebSocket in order to ensure URLs they operate on are rewritten.
*
* @see https://xhr.spec.whatwg.org/
* @see https://fetch.spec.whatwg.org/
* @see https://html.spec.whatwg.org/multipage/web-sockets.html#websocket
* @see https://html.spec.whatwg.org/multipage/server-sent-events.html#the-eventsource-interface
*/
WBWombat.prototype.initHTTPOverrides = function() {
var wombat = this;
if (
self.XMLHttpRequest &&
self.XMLHttpRequest.prototype &&
self.XMLHttpRequest.prototype.open
) {
var oXHROpen = self.XMLHttpRequest.prototype.open;
self.XMLHttpRequest.prototype.open = function open(
method,
url,
async,
user,
password
) {
var rwURL = wombat.rewriteURL(url);
var openAsync = true;
if (async != null && !async) openAsync = false;
oXHROpen.call(this, method, rwURL, openAsync, user, password);
if (rwURL.indexOf('data:') === -1) {
this.setRequestHeader('X-Pywb-Requested-With', 'XMLHttpRequest');
}
};
}
if (self.fetch != null) {
// this fetch is Worker.fetch
var orig_fetch = self.fetch;
self.fetch = function fetch(input, init_opts) {
var rwInput = wombat.rewriteFetchApi(input);
var newInitOpts = init_opts || {};
newInitOpts['credentials'] = 'include';
return orig_fetch.call(this, rwInput, newInitOpts);
};
}
if (self.Request && self.Request.prototype) {
var orig_request = self.Request;
self.Request = (function(Request_) {
return function Request(input, init_opts) {
var newInitOpts = init_opts || {};
var newInput = wombat.rewriteFetchApi(input);
newInitOpts['credentials'] = 'include';
return new Request_(newInput, newInitOpts);
};
})(self.Request);
self.Request.prototype = orig_request.prototype;
}
if (self.Response && self.Response.prototype) {
var originalRedirect = self.Response.prototype.redirect;
self.Response.prototype.redirect = function redirect(url, status) {
var rwURL = wombat.rewriteUrl(url);
return originalRedirect.call(this, rwURL, status);
};
}
if (self.EventSource && self.EventSource.prototype) {
var origEventSource = self.EventSource;
self.EventSource = (function(EventSource_) {
return function EventSource(url, configuration) {
var rwURL = url;
if (url != null) {
rwURL = wombat.rewriteUrl(url);
}
return new EventSource_(rwURL, configuration);
};
})(self.EventSource);
self.EventSource.prototype = origEventSource.prototype;
Object.defineProperty(self.EventSource.prototype, 'constructor', {
value: self.EventSource
});
}
if (self.WebSocket && self.WebSocket.prototype) {
var origWebSocket = self.WebSocket;
self.WebSocket = (function(WebSocket_) {
return function WebSocket(url, configuration) {
var rwURL = url;
if (url != null) {
rwURL = wombat.rewriteWSURL(url);
}
return new WebSocket_(rwURL, configuration);
};
})(self.WebSocket);
self.WebSocket.prototype = origWebSocket.prototype;
Object.defineProperty(self.WebSocket.prototype, 'constructor', {
value: self.WebSocket
});
}
};
/**
* Applies an override to Clients.openWindow and WindowClient.navigate that rewrites
* the supplied URL that represents a controlled window
* @see https://w3c.github.io/ServiceWorker/#window-client-interface
* @see https://w3c.github.io/ServiceWorker/#clients-interface
*/
WBWombat.prototype.initClientApisOverride = function() {
var wombat = this;
if (
self.Clients &&
self.Clients.prototype &&
self.Clients.prototype.openWindow
) {
var oClientsOpenWindow = self.Clients.prototype.openWindow;
self.Clients.prototype.openWindow = function openWindow(url) {
var rwURL = wombat.rewriteClientWindowURL(url);
return oClientsOpenWindow.call(this, rwURL);
};
}
if (
self.WindowClient &&
self.WindowClient.prototype &&
self.WindowClient.prototype.navigate
) {
var oWinClientNavigate = self.WindowClient.prototype.navigate;
self.WindowClient.prototype.navigate = function navigate(url) {
var rwURL = wombat.rewriteClientWindowURL(url, this);
return oWinClientNavigate.call(this, rwURL);
};
}
};
/**
* Applies overrides to the CachStorage and Cache interfaces in order
* to rewrite the URLs they operate on
* @see https://w3c.github.io/ServiceWorker/#cachestorage
* @see https://w3c.github.io/ServiceWorker/#cache-interface
*/
WBWombat.prototype.initCacheApisOverride = function() {
var wombat = this;
if (
self.CacheStorage &&
self.CacheStorage.prototype &&
self.CacheStorage.prototype.match
) {
var oCacheStorageMatch = self.CacheStorage.prototype.match;
self.CacheStorage.prototype.match = function match(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheStorageMatch.call(this, rwRequest, options);
};
}
if (self.Cache && self.Cache.prototype) {
if (self.Cache.prototype.match) {
var oCacheMatch = self.Cache.prototype.match;
self.Cache.prototype.match = function match(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheMatch.call(this, rwRequest, options);
};
}
if (self.Cache.prototype.matchAll) {
var oCacheMatchAll = self.Cache.prototype.matchAll;
self.Cache.prototype.matchAll = function matchAll(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheMatchAll.call(this, rwRequest, options);
};
}
if (self.Cache.prototype.add) {
var oCacheAdd = self.Cache.prototype.add;
self.Cache.prototype.add = function add(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheAdd.call(this, rwRequest, options);
};
}
if (self.Cache.prototype.addAll) {
var oCacheAddAll = self.Cache.prototype.addAll;
self.Cache.prototype.addAll = function addAll(requests) {
var rwRequests = requests;
if (Array.isArray(requests)) {
rwRequests = new Array(requests.length);
for (var i = 0; i < requests.length; i++) {
rwRequests[i] = wombat.rewriteCacheApi(requests[i]);
}
}
return oCacheAddAll.call(this, rwRequests);
};
}
if (self.Cache.prototype.put) {
var oCachePut = self.Cache.prototype.put;
self.Cache.prototype.put = function put(request, response) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCachePut.call(this, rwRequest, response);
};
}
if (self.Cache.prototype.delete) {
var oCacheDelete = self.Cache.prototype.delete;
self.Cache.prototype.delete = function newCacheDelete(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheDelete.call(this, rwRequest, options);
};
}
if (self.Cache.prototype.keys) {
var oCacheKeys = self.Cache.prototype.keys;
self.Cache.prototype.keys = function keys(request, options) {
var rwRequest = wombat.rewriteCacheApi(request);
return oCacheKeys.call(this, rwRequest, options);
};
}
}
};
self.WBWombat = WBWombat;

25
wombat/test/assets/it.html Normal file
View File

@ -0,0 +1,25 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>this is it</title>
<style>
html,
body {
height: 100%;
width: 100%;
margin: 0;
padding: 0;
border: 0;
overflow: hidden;
}
</style>
</head>
<body>
<p
style="top: 50%; left: 50%; transform: translate(-50%, -50%); font-size: x-large; position: fixed;"
>
This is it y'all, you found it!
</p>
</body>
</html>

40
wombat/test/assets/sandbox.html Executable file
View File

@ -0,0 +1,40 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>Wombat sandbox</title>
<script>
window.wombatSandbox = {
window,
document,
originalLocation: location.href
};
window.wbinfo = {};
window.wbinfo.enable_auto_fetch = false;
window.wbinfo.top_url = location.href.replace('mp_', '');
window.wbinfo.url = 'https://tests.wombat.io/';
window.wbinfo.timestamp = '20180803160549';
window.wbinfo.request_ts = '20180803160549';
window.wbinfo.prefix = decodeURI(
`${window.location.protocol}//localhost:${window.location.port}/live/`
);
window.wbinfo.mod = 'mp_';
window.wbinfo.is_framed = true;
window.wbinfo.is_live = false;
window.wbinfo.coll = '';
window.wbinfo.proxy_magic = '';
window.wbinfo.static_prefix =
'https://content.webrecorder.io/static/bundle/';
window.wbinfo.wombat_ts = '20180803160549';
window.wbinfo.wombat_sec = '1533312349';
window.wbinfo.wombat_scheme = 'https';
window.wbinfo.wombat_host = 'tests.wombat.io';
window.wbinfo.wombat_opts = {};
window.wbinfo.state = '';
window.wbinfo.metadata = {};
</script>
</head>
<body>
<script src="/wombat.js"></script>
</body>
</html>

193
wombat/test/assets/testPage.html Normal file
View File

@ -0,0 +1,193 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>Test Page</title>
<link rel="stylesheet" href="./bootstrap.min.css" />
<style>
#wombatSandbox {
width: 0;
height: 0;
margin: 0;
padding: 0;
border: 0;
overflow: hidden;
}
.bottomBoarder {
border-bottom: 1px solid rgba(0, 0, 0, 0.125);
}
</style>
<script src="./testPageBundle.js"></script>
</head>
<body>
<div class="container mt-2">
<div class="row">
<div class="col-12">
<h5 class="text-center">Messages sent by wombat</h5>
</div>
</div>
<div class="row mt-4 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Load:</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="load-url"><b>url:</b></p>
</div>
</div>
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="load-title"><b>title:</b></p>
</div>
</div>
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="load-readyState"><b>readyState:</b></p>
</div>
</div>
</div>
</div>
<div class="row mt-1 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Replace URL:</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="replace-url-url"><b>url:</b></p>
</div>
</div>
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="replace-url-title"><b>title:</b></p>
</div>
</div>
</div>
</div>
<div class="row mt-1 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Cookie:</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="cookie-domain"><b>domain:</b></p>
</div>
</div>
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="cookie-cookie"><b>cookie:</b></p>
</div>
</div>
</div>
</div>
<div class="row mt-1 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Title</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="title-title"><b>title:</b></p>
</div>
</div>
</div>
</div>
<div class="row mt-1 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Hash Change</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="hash-hash"><b>hash:</b></p>
</div>
</div>
</div>
</div>
<div class="row mt-1 align-items-center bottomBoarder pb-1">
<div class="col-2 align-self-center">
<h5 class="mb-0 text-center">Unknown</h5>
</div>
<div class="col-10 h-100">
<div class="row">
<div class="col-12">
<p class="mb-0 lead" id="unknown-msg"></p>
</div>
</div>
</div>
</div>
</div>
<iframe
id="wombatSandbox"
src="/live/20180803160549mp_/https://tests.wombat.io/"
></iframe>
<script>
const defaultText = ' not sent';
const domStructure = {
sandbox: document.getElementById('wombatSandbox'),
load: {
url: document.createTextNode(defaultText),
title: document.createTextNode(defaultText),
readyState: document.createTextNode(defaultText),
reset() {
this.url.data = defaultText;
this.title.data = defaultText;
this.readyState.data = defaultText;
}
},
replaceURL: {
url: document.createTextNode(defaultText),
title: document.createTextNode(defaultText),
reset() {
this.url.data = defaultText;
this.title.data = defaultText;
}
},
cookie: {
domain: document.createTextNode(defaultText),
cookie: document.createTextNode(defaultText),
reset() {
this.cookie.data = defaultText;
this.domain.data = defaultText;
}
},
titleMsg: document.createTextNode(defaultText),
hashchange: document.createTextNode(defaultText),
unknown: document.createTextNode(defaultText),
reset() {
this.load.reset();
this.replaceURL.reset();
this.cookie.reset();
this.titleMsg.data = defaultText;
this.hashchange.data = defaultText;
this.unknown.data = defaultText;
}
};
document.getElementById('load-url').appendChild(domStructure.load.url);
document
.getElementById('load-title')
.appendChild(domStructure.load.title);
document
.getElementById('load-readyState')
.appendChild(domStructure.load.readyState);
document
.getElementById('replace-url-url')
.appendChild(domStructure.replaceURL.url);
document
.getElementById('replace-url-title')
.appendChild(domStructure.replaceURL.title);
document
.getElementById('cookie-cookie')
.appendChild(domStructure.cookie.cookie);
document
.getElementById('cookie-domain')
.appendChild(domStructure.cookie.domain);
document.getElementById('title-title').appendChild(domStructure.titleMsg);
document.getElementById('hash-hash').appendChild(domStructure.hashchange);
document.getElementById('unknown-msg').appendChild(domStructure.unknown);
window.overwatch = new TestOverwatch(domStructure);
</script>
</body>
</html>

6
wombat/test/assets/testWorker.js Normal file
View File

@ -0,0 +1,6 @@
self.isFetchOverridden = () =>
self.fetch.toString().includes('rewriteFetchApi');
self.isImportScriptOverridden = () =>
self.importScripts.toString().includes('rewriteArgs');
self.isAjaxRewritten = () =>
self.XMLHttpRequest.prototype.open.toString().includes('rewriteURL');

11
wombat/test/helpers/cert.pem Normal file
View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBljCCATygAwIBAgIUZy3QF0k1ohnxIYNtmZAdPVKzToAwCgYIKoZIzj0EAwIw
FDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE5MDMwNDE2NDk0NloXDTIwMDMwMzE2
NDk0NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D
AQcDQgAEgQ3XpSPtkrLsmlFG6VxfHx/LrYEsp2G7VuceO9uWsp10qUHTMoV1T/6u
yJn5R9Uax6aDfDGDHtzbjimIBAUv/6NsMGowaAYDVR0RBGEwX4IJbG9jYWxob3N0
ggsqLmxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluhwR/AAABhwQAAAAA
hxAAAAAAAAAAAAAAAAAAAAABhxAAAAAAAAAAAAAAAAAAAAAAMAoGCCqGSM49BAMC
A0gAMEUCIALCxk8CM8uJxPx35glfIS8+xTzzUjfkJmPNSY+gUuh/AiEA+VeoJWQL
FRgURnAyaS3eCoY8plnEAO5OmmhBrlOghU4=
-----END CERTIFICATE-----

168
wombat/test/helpers/initChrome.js Normal file
View File

@ -0,0 +1,168 @@
const cp = require('child_process');
const path = require('path');
const os = require('os');
const fs = require('fs-extra');
const readline = require('readline');
const chromeFinder = require('chrome-launcher/dist/chrome-finder');
const criHelper = require('chrome-remote-interface-extra/lib/helper');
const CHROME_PROFILE_PATH = path.join(os.tmpdir(), 'temp_chrome_profile-');
const winPos = !process.env.NO_MOVE_WINDOW ? '--window-position=2000,0' : '';
const chromeArgs = userDataDir => [
'--enable-automation',
'--force-color-profile=srgb',
'--remote-debugging-port=9222',
'--disable-background-networking',
'--disable-background-timer-throttling',
'--disable-renderer-backgrounding',
'--disable-backgrounding-occluded-windows',
'--disable-ipc-flooding-protection',
'--enable-features=NetworkService,NetworkServiceInProcess',
'--disable-client-side-phishing-detection',
'--disable-default-apps',
'--disable-extensions',
'--disable-popup-blocking',
'--disable-hang-monitor',
'--disable-prompt-on-repost',
'--disable-sync',
'--disable-domain-reliability',
'--disable-infobars',
'--disable-features=site-per-process,TranslateUI',
'--disable-breakpad',
'--disable-backing-store-limit',
'--metrics-recording-only',
'--no-first-run',
'--safebrowsing-disable-auto-update',
'--password-store=basic',
'--use-mock-keychain',
'--mute-audio',
'--autoplay-policy=no-user-gesture-required',
`--user-data-dir=${userDataDir}`,
winPos,
'about:blank'
];
const preferredExes = {
linux: [
'google-chrome-unstable',
'google-chrome-beta',
'google-chrome-stable'
],
darwin: ['Chrome Canary', 'Chrome'],
win32: []
};
function findChrome() {
const findingFn = chromeFinder[process.platform];
if (findingFn == null) {
throw new Error(
`Can not find chrome exe, unsupported platform - ${process.platform}`
);
}
const exes = findingFn();
const preferred = preferredExes[process.platform] || [];
let exe;
for (let i = 0; i < preferred.length; i++) {
exe = exes.find(anExe => anExe.includes(preferred[i]));
if (exe) return exe;
}
return exes[0];
}
/**
*
* @return {Promise<{chromeProcess: ChildProcess, killChrome: function(): void}>}
*/
async function initChrome() {
const executable = findChrome();
const userDataDir = await fs.mkdtemp(CHROME_PROFILE_PATH);
const chromeArguments = chromeArgs(userDataDir);
const chromeProcess = cp.spawn(executable, chromeArguments, {
stdio: ['ignore', 'ignore', 'pipe'],
env: process.env,
detached: process.platform !== 'win32'
});
const maybeRemoveUDataDir = () => {
try {
fs.removeSync(userDataDir);
} catch (e) {}
};
let killed = false;
const killChrome = () => {
if (killed) {
return;
}
killed = true;
chromeProcess.kill('SIGKILL');
// process.kill(-chromeProcess.pid, 'SIGKILL')
maybeRemoveUDataDir();
};
process.on('exit', killChrome);
chromeProcess.once('exit', maybeRemoveUDataDir);
process.on('SIGINT', () => {
killChrome();
process.exit(130);
});
process.once('SIGTERM', killChrome);
process.once('SIGHUP', killChrome);
await waitForWSEndpoint(chromeProcess, 15 * 1000);
return { chromeProcess, killChrome };
}
module.exports = initChrome;
// module.exports = initChrome
function waitForWSEndpoint(chromeProcess, timeout) {
return new Promise((resolve, reject) => {
const rl = readline.createInterface({ input: chromeProcess.stderr });
let stderr = '';
const listeners = [
criHelper.helper.addEventListener(rl, 'line', onLine),
criHelper.helper.addEventListener(rl, 'close', onClose),
criHelper.helper.addEventListener(chromeProcess, 'exit', onClose),
criHelper.helper.addEventListener(chromeProcess, 'error', onClose)
];
const timeoutId = timeout ? setTimeout(onTimeout, timeout) : 0;
function onClose() {
cleanup();
reject(new Error(['Failed to launch chrome!', stderr].join('\n')));
}
function onTimeout() {
cleanup();
reject(
new Error(
`Timed out after ${timeout} ms while trying to connect to Chrome!`
)
);
}
/**
* @param {string} line
*/
function onLine(line) {
stderr += line + '\n';
const match = line.match(/^DevTools listening on (ws:\/\/.*)$/);
if (!match) {
return;
}
cleanup();
resolve(match[1]);
}
function cleanup() {
if (timeoutId) {
clearTimeout(timeoutId);
}
criHelper.helper.removeEventListeners(listeners);
rl.close();
}
});
}

144
wombat/test/helpers/initServer.js Normal file
View File

@ -0,0 +1,144 @@
const path = require('path');
const fs = require('fs-extra');
const createServer = require('fastify');
const host = '127.0.0.1';
const port = 3030;
const gracefullShutdownTimeout = 50000;
const shutdownOnSignals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
const assetsPath = path.join(__dirname, '..', 'assets');
const httpsSandboxPath = path.join(assetsPath, 'sandbox.html');
const theyFoundItPath = path.join(assetsPath, 'it.html');
function promiseResolveReject() {
const prr = { promise: null, resolve: null, reject: null };
prr.promise = new Promise((resolve, reject) => {
let to = setTimeout(
() => reject(new Error('wait for request timed-out')),
15000
);
prr.resolve = () => {
clearTimeout(to);
resolve();
};
prr.reject = reason => {
clearTimeout(to);
reject(reason);
};
});
return prr;
}
/**
* @return {Promise<fastify.FastifyInstance>}
*/
async function initServer() {
const serverOpts = { logger: false };
const requestSubscribers = new Map();
const checkReqSubscribers = (pathName, request, reply) => {
const handler = requestSubscribers.get(pathName);
if (handler) {
handler.resolve(request);
requestSubscribers.delete(pathName);
}
};
const fastify = createServer(serverOpts);
fastify
.get(
'/live/20180803160549wkr_/https://tests.wombat.io/testWorker.js',
async (request, reply) => {
const init = `new WBWombat({'prefix': 'http://localhost:${port}/live/20180803160549', 'prefixMod': 'http://localhost:${port}/live/20180803160549wkr_/', 'originalURL': 'https://tests.wombat.io/testWorker.js'});`;
reply
.code(200)
.type('application/javascript; charset=UTF-8')
.send(
`self.importScripts('/testWorker.js');(function() { self.importScripts('/wombatWorkers.js'); ${init}})();`
);
}
)
.get(
'/live/20180803160549sw_/https://tests.wombat.io/testServiceWorker.js',
(request, reply) => {
reply
.code(200)
.type('application/javascript; charset=UTF-8')
.header(
'Service-Worker-Allowed',
`${address}/live/20180803160549mp_/https://tests.wombat.io/`
)
.send('console.log("hi")');
}
)
.get(
'/live/20180803160549mp_/https://tests.wombat.io/it',
async (request, reply) => {
reply.type('text/html').status(200);
return fs.createReadStream(theyFoundItPath);
}
)
.get(
'/live/20180803160549mp_/https://tests.wombat.io/',
async (request, reply) => {
reply.type('text/html').status(200);
return fs.createReadStream(httpsSandboxPath);
}
)
.get(
'/live/20180803160549mp_/https://tests.wombat.io/test',
async (request, reply) => {
reply.type('application/json; charset=utf-8').status(200);
return { headers: request.headers, url: request.raw.originalUrl };
}
)
.decorate('reset', () => {
const error = new Error('Static Server has been reset');
for (const prr of requestSubscribers.values()) {
prr.reject.call(null, error);
}
requestSubscribers.clear();
})
.decorate('stop', () => {
fastify.reset();
return fastify.close();
})
.decorate('testPage', `http://localhost:${port}/testPage.html`)
.decorate('waitForRequest', route => {
let prr = requestSubscribers.get(route);
if (prr) return prr.promise;
prr = promiseResolveReject();
requestSubscribers.set(route, prr);
return prr.promise;
})
.addHook('onRequest', (request, reply, next) => {
checkReqSubscribers(request.raw.url, request, reply);
// console.log(`${request.raw.method} ${request.raw.url}`);
next();
})
.register(require('fastify-favicon'))
.register(require('fastify-static'), {
root: assetsPath,
etag: false,
lastModified: false
});
shutdownOnSignals.forEach(signal => {
process.once(signal, () => {
setTimeout(() => {
console.error(
`received ${signal} signal, terminate process after timeout of ${gracefullShutdownTimeout}ms`
);
process.exit(1);
}, gracefullShutdownTimeout).unref();
console.log(`received ${signal} signal, triggering close hook`);
fastify.stop().then(() => {
process.exit(0);
});
});
});
const address = await fastify.listen(port, host);
return fastify;
}
module.exports = initServer;

5
wombat/test/helpers/key.pem Normal file
View File

@ -0,0 +1,5 @@
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgJdKmOQ1C7CuBwPsM
VL44Pr99/lRqHnMi9+YCWSggsXWhRANCAASBDdelI+2SsuyaUUbpXF8fH8utgSyn
YbtW5x4725aynXSpQdMyhXVP/q7ImflH1RrHpoN8MYMe3NuOKYgEBS//
-----END PRIVATE KEY-----

154
wombat/test/helpers/testHelper.js Normal file
View File

@ -0,0 +1,154 @@
const initChrome = require('./initChrome');
const initServer = require('./initServer');
const { CRIExtra, Browser, Events } = require('chrome-remote-interface-extra');
const testDomains = { workers: true };
class TestHelper {
/**
* @param {*} t
* @return {Promise<TestHelper>}
*/
static async init(t) {
const { chromeProcess, killChrome } = await initChrome();
const server = await initServer();
const { webSocketDebuggerUrl } = await CRIExtra.Version();
const client = await CRIExtra({ target: webSocketDebuggerUrl });
const browser = await Browser.create(client, {
ignoreHTTPSErrors: true,
process: chromeProcess,
additionalDomains: testDomains,
async closeCallback() {
killChrome();
}
});
await browser.waitForTarget(t => t.type() === 'page');
const th = new TestHelper({ server, client, browser, t, killChrome });
await th.setup();
return th;
}
/**
* @param {TestHelperInit} init
*/
constructor({ server, client, browser, t, killChrome }) {
/**
* @type {fastify.FastifyInstance}
*/
this._server = server;
/**
* @type {CRIConnection}
*/
this._client = client;
/**
* @type {Browser}
*/
this._browser = browser;
/** @type {*} */
this._t = t;
this._killChrome = killChrome;
/** @type {Page} */
this._testPage = null;
/** @type {Frame} */
this._sandbox = null;
}
/**
* @return {fastify.FastifyInstance}
*/
server() {
return this._server;
}
/**
* @return {Page}
*/
testPage() {
return this._testPage;
}
/**
* @return {Frame}
*/
sandbox() {
return this._sandbox;
}
async initWombat() {
await this._testPage.evaluate(() => {
window.overwatch.initSandbox();
});
}
async maybeInitWombat() {
await this._testPage.evaluate(() => {
window.overwatch.maybeInitSandbox();
});
}
async setup() {
this._testPage = await this._browser.newPage();
await this.cleanup();
}
async cleanup() {
await this._testPage.goto(this._server.testPage, {
waitUntil: 'networkidle2'
});
this._sandbox = this._testPage.frames()[1];
}
async fullRefresh() {
await this.cleanup();
await this.initWombat();
}
async ensureSandbox() {
if (!this._sandbox.url().endsWith('https://tests.wombat.io/')) {
await this.fullRefresh();
} else {
await this.maybeInitWombat();
}
}
async stop() {
if (this._testPage) {
try {
await this._testPage.close();
} catch (e) {
console.log(`Exception closing test page ${e}`);
}
}
try {
if (this._browser) {
await this._browser.close();
}
} catch (e) {
console.log(`Exception closing browser ${e}`);
}
try {
if (this._server) {
await this._server.stop();
}
} catch (e) {
console.log(`Exception stopping server ${e}`);
}
}
}
module.exports = TestHelper;
/**
* @typedef {Object} TestHelperInit
* @property {Browser} browser
* @property {CRIConnection} client
* @property {fastify.FastifyInstance} server
* @property {*} t
* @property {function(): void} killChrome
*/

1066
wombat/test/helpers/testedValues.js Normal file
View File

File diff suppressed because it is too large Load Diff

17
wombat/test/helpers/utils.js Normal file
View File

@ -0,0 +1,17 @@
const { URL } = require('url');
exports.delay = howMuch =>
new Promise(resolve => {
setTimeout(resolve, howMuch);
});
const extractModifier = /\/live\/[0-9]+([a-z_]+)\/.+/;
exports.extractModifier = rwURL => {
const purl = new URL(rwURL);
const result = extractModifier.exec(purl.pathname);
if (result) return result[1];
return null;
};
exports.parsedURL = url => new URL(url);

76
wombat/test/original-karma-tests.js Normal file
View File

@ -0,0 +1,76 @@
import test from 'ava';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.server = helper.server();
t.context.testPage = helper.testPage();
});
test.afterEach.always(async t => {
await helper.ensureSandbox();
});
test.after.always(async t => {
await helper.stop();
});
test('anchor rewriting - should rewrite links in dynamically injected <a> tags', async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(() => {
document.body.innerHTML = '<a href="foobar.html" id="link">A link</a>';
const a = document.getElementById('link');
if (a == null) return { exists: false };
const result = { exists: true, href: a.href };
a.remove();
return result;
});
t.deepEqual(result, {
exists: true,
href: 'https://tests.wombat.io/foobar.html'
});
});
test('anchor rewriting - should rewrite links in dynamically injected <a> tags and toString() should return the resolved original URL', async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(() => {
document.body.innerHTML = '<a href="foobar.html" id="link">A link</a>';
const a = document.getElementById('link');
if (a == null) return { exists: false };
a.remove();
return { exists: true, toString: a.toString() };
});
t.deepEqual(result, {
exists: true,
toString: 'https://tests.wombat.io/foobar.html'
});
});
test('base URL overrides - document.baseURI should return the original URL', async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(() => {
var baseURI = document.baseURI;
return { type: typeof baseURI, baseURI };
});
t.deepEqual(result, { type: 'string', baseURI: 'https://tests.wombat.io/' });
});
test('base URL overrides - should allow base.href to be assigned', async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(() => {
var baseElement = document.createElement('base');
baseElement.href = 'http://foobar.com/base';
return baseElement.href;
});
t.is(result, 'http://foobar.com/base');
});

256
wombat/test/overrides-browser.js Normal file
View File

@ -0,0 +1,256 @@
import test from 'ava';
import { URLParts, WB_PREFIX } from './helpers/testedValues';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.server = helper.server();
t.context.testPage = helper.testPage();
});
test.afterEach.always(async t => {
if (t.title.includes('SharedWorker')) {
await helper.fullRefresh();
} else {
await helper.ensureSandbox();
}
});
test.after.always(async t => {
await helper.stop();
});
test('The actual top should have been sent the loadMSG', async t => {
const { testPage, server } = t.context;
const result = await testPage.evaluate(
() => window.overwatch.wbMessages.load
);
t.true(
result,
'The message sent by wombat to inform top it has loaded should have been sent'
);
});
test('init_top_frame: should set __WB_replay_top correctly', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.__WB_replay_top === window
);
t.true(result, 'The replay top should equal to frames window object');
});
test('init_top_frame: should set __WB_orig_parent correctly', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.__WB_orig_parent === window.top
);
t.true(result, '__WB_orig_parent should equal the actual top');
});
test('init_top_frame: should set parent to itself (__WB_replay_top)', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.parent === window.__WB_replay_top
);
t.true(result, 'window.parent should equal to itself (__WB_replay_top)');
});
test('WombatLocation: should be added to window as WB_wombat_location', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.WB_wombat_location != null
);
t.true(result, 'WB_wombat_location was not added to window');
});
for (let i = 0; i < URLParts.length; i++) {
const urlPart = URLParts[i];
test(`WombatLocation: should make available '${urlPart}'`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
upart => window.WB_wombat_location[upart] != null,
urlPart
);
t.true(result, `WB_wombat_location does not make available '${urlPart}'`);
});
test(`WombatLocation: the '${urlPart}' property should be equal to the same value as would be returned on the live web`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
upart =>
new URL(window.wbinfo.url)[upart] === window.WB_wombat_location[upart],
urlPart
);
t.true(
result,
`WB_wombat_location return a value equal to the original for '${urlPart}'`
);
});
}
test('WombatLocation: should return the href property as the value for toString', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.WB_wombat_location.toString() === window.wbinfo.url
);
t.true(
result,
`WB_wombat_location does not return the href property as the value for toString`
);
});
test('WombatLocation: should return itself as the value for valueOf', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.WB_wombat_location.valueOf() === window.WB_wombat_location
);
t.true(
result,
`WB_wombat_location does not return itself as the value for valueOf`
);
});
test('WombatLocation: should have a Symbol.toStringTag value of "Location"', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() =>
window.WB_wombat_location[window.Symbol.toStringTag] ===
location[window.Symbol.toStringTag]
);
t.true(
result,
`WB_wombat_location does not have a Symbol.toStringTag value of "Location"`
);
});
test('WombatLocation browser navigation control: should rewrite Location.replace usage', async t => {
const { sandbox, server } = t.context;
const [navigationResponse] = await Promise.all([
sandbox.waitForNavigation(),
sandbox.evaluate(() => {
window.WB_wombat_location.replace('/it');
})
]);
t.is(
navigationResponse.url(),
`${WB_PREFIX}mp_/https://tests.wombat.io/it`,
'using WB_wombat_location.replace did not navigate the page'
);
});
test('WombatLocation browser navigation control: should rewrite Location.assign usage', async t => {
const { sandbox, server } = t.context;
const [navigationResponse] = await Promise.all([
sandbox.waitForNavigation(),
sandbox.evaluate(() => {
window.WB_wombat_location.assign('/it');
})
]);
t.is(
navigationResponse.url(),
`${WB_PREFIX}mp_/https://tests.wombat.io/it`,
'using WB_wombat_location.assign did not navigate the page'
);
});
test('WombatLocation browser navigation control: should reload the page via Location.reload usage', async t => {
const { sandbox, server } = t.context;
const [originalLoc, navigationResponse] = await Promise.all([
sandbox.evaluate(() => window.location.href),
sandbox.waitForNavigation(),
sandbox.evaluate(() => {
window.WB_wombat_location.reload();
})
]);
t.is(
navigationResponse.url(),
originalLoc,
'using WB_wombat_location.reload did not reload the page'
);
});
test('browser history control: should rewrite history.pushState', async t => {
const { sandbox, server } = t.context;
const [originalLoc, newloc] = await Promise.all([
sandbox.evaluate(() => window.location.href),
sandbox.evaluate(() => {
window.history.pushState(null, null, '/it');
return window.location.href;
})
]);
t.is(
newloc,
`${originalLoc}it`,
'history navigations using pushState are not rewritten'
);
const result = await sandbox.evaluate(
() => window.WB_wombat_location.href === 'https://tests.wombat.io/it'
);
t.true(
result,
'WB_wombat_location.href does not update after history.pushState usage'
);
});
test('browser history control: should rewrite history.replaceState', async t => {
const { sandbox, server } = t.context;
const [originalLoc, newloc] = await Promise.all([
sandbox.evaluate(() => window.location.href),
sandbox.evaluate(() => {
window.history.replaceState(null, null, '/it2');
return window.location.href;
})
]);
t.is(
newloc,
`${originalLoc}it2`,
'history navigations using pushState are not rewritten'
);
const result = await sandbox.evaluate(
() => window.WB_wombat_location.href === 'https://tests.wombat.io/it2'
);
t.true(
result,
'WB_wombat_location.href does not update after history.replaceState usage'
);
});
test('browser history control: should send the "replace-url" msg to the top frame on history.pushState usage', async t => {
const { sandbox, testPage } = t.context;
await sandbox.evaluate(() => window.history.pushState(null, null, '/it3'));
const result = await testPage.evaluate(
() =>
window.overwatch.wbMessages['replace-url'].url != null &&
window.overwatch.wbMessages['replace-url'].url ===
'https://tests.wombat.io/it3'
);
t.true(
result,
'the "replace-url" message was not sent to the top frame on history.pushState usage'
);
});
test('browser history control: should send the "replace-url" msg to the top frame on history.replaceState usage', async t => {
const { sandbox, testPage } = t.context;
await sandbox.evaluate(() => window.history.replaceState(null, null, '/it4'));
t.true(
await testPage.evaluate(
() =>
window.overwatch.wbMessages['replace-url'].url != null &&
window.overwatch.wbMessages['replace-url'].url ===
'https://tests.wombat.io/it4'
),
'the "replace-url" message was not sent to the top frame on history.pushState usage'
);
});

212
wombat/test/overrides-css.js Normal file
View File

@ -0,0 +1,212 @@
import test from 'ava';
import { CSS } from './helpers/testedValues';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.server = helper.server();
t.context.testPage = helper.testPage();
});
test.afterEach.always(async t => {
await helper.ensureSandbox();
});
test.after.always(async t => {
await helper.stop();
});
for (const attrToProp of CSS.styleAttrs.attrs) {
test(`style.${attrToProp.attr}: assignments should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNAttr,
attrToProp
);
t.notDeepEqual(result, CSS.styleAttrs.unrw);
});
}
for (const attrToProp of CSS.styleAttrs.attrs) {
test(`style["${
attrToProp.attr
}"]: assignments should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNPropName,
attrToProp
);
t.notDeepEqual(result, CSS.styleAttrs.unrw);
});
if (attrToProp.attr !== attrToProp.propName) {
test(`style["${
attrToProp.propName
}"]: assignments should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNPropName,
attrToProp
);
t.notDeepEqual(result, CSS.styleAttrs.unrw);
});
}
}
for (const attrToProp of CSS.styleAttrs.attrs) {
test(`style.setProperty("${
attrToProp.attr
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNSetProp,
attrToProp.attr,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
if (attrToProp.attr !== attrToProp.propName) {
test(`style.setProperty("${
attrToProp.propName
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNSetProp,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
}
}
for (const attrToProp of CSS.styleAttrs.attrs) {
test(`style.cssText: assignments of '${
attrToProp.propName
}' should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleAttrs.testFNCssText,
attrToProp
);
t.notDeepEqual(result, attrToProp.unrw);
});
}
for (const aTest of CSS.styleTextContent.tests) {
test(`style.textContent: assignments using an css definitions containing '${
aTest.name
}' should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.styleTextContent.testFN,
aTest.unrw
);
t.notDeepEqual(result, aTest.unrw);
});
}
for (const aTest of CSS.StyleSheetInsertRule.tests) {
test(`CSSStyleSheet.insertRule: inserting a new rule containing '${
aTest.name
}' should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.StyleSheetInsertRule.testFN,
aTest.unrw
);
t.notDeepEqual(result, aTest.unrw);
});
}
for (const aTest of CSS.CSSRuleCSSText.tests) {
test(`CSSRule.cssText: modifying an existing rule to become a new rule containing '${
aTest.name
}' should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.CSSRuleCSSText.testFN,
aTest.unrw
);
t.notDeepEqual(result, aTest.unrw);
});
}
for (const attrToProp of CSS.StylePropertyMap.tests) {
test(`StylePropertyMap.set("${
attrToProp.attr
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.StylePropertyMap.testFNSet,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
if (!CSS.StylePropertyMap.noAppend.has(attrToProp.attr)) {
test(`StylePropertyMap.append("${
attrToProp.attr
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.StylePropertyMap.testFNAppend,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
}
}
for (const attrToProp of CSS.CSSKeywordValue.tests) {
test(`new CSSKeywordValue("${
attrToProp.propName
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.CSSKeywordValue.testFN,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
}
for (const attrToProp of CSS.CSSStyleValue.tests) {
if (CSS.CSSStyleValue.skipped.has(attrToProp.attr)) continue;
test(`CSSStyleValue.parse("${
attrToProp.propName
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.CSSStyleValue.testFNParse,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
test(`CSSStyleValue.parseAll("${
attrToProp.propName
}", "value"): value should be rewritten`, async t => {
const { sandbox } = t.context;
const result = await sandbox.evaluate(
CSS.CSSStyleValue.testFNParseAll,
attrToProp.propName,
attrToProp.unrw
);
t.notDeepEqual(result, attrToProp.unrw);
});
}

1001
wombat/test/overrides-dom.js Normal file
View File

File diff suppressed because it is too large Load Diff

159
wombat/test/overrides-http.js Normal file
View File

@ -0,0 +1,159 @@
import test from 'ava';
import { mpURL } from './helpers/testedValues';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.server = helper.server();
t.context.testPage = helper.testPage();
});
test.afterEach.always(async t => {
if (t.title.includes('SharedWorker')) {
await helper.fullRefresh();
} else {
await helper.ensureSandbox();
}
});
test.after.always(async t => {
await helper.stop();
});
test('XMLHttpRequest: should rewrite the URL argument of "open"', async t => {
const { sandbox, server } = t.context;
const response = await sandbox.evaluate(async () => {
let reqDone;
let to;
const prom = new Promise(resolve => {
reqDone = resolve;
to = setTimeout(() => resolve(false), 5000);
});
const onLoad = () => {
clearTimeout(to);
reqDone(true);
};
const xhr = new XMLHttpRequest();
xhr.addEventListener('load', onLoad);
xhr.open('GET', '/test');
xhr.send();
const loaded = await prom;
if (!loaded) throw new Error('no reply from server in 5 seconds');
return JSON.parse(xhr.responseText);
});
t.is(response.headers['x-pywb-requested-with'], 'XMLHttpRequest');
t.is(response.url, '/live/20180803160549mp_/https://tests.wombat.io/test');
});
test('XMLHttpRequest: should rewrite the "responseURL" property', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(async () => {
let reqDone;
let to;
const prom = new Promise(resolve => {
reqDone = resolve;
to = setTimeout(() => resolve(false), 5000);
});
const onLoad = () => {
clearTimeout(to);
reqDone(true);
};
const xhr = new XMLHttpRequest();
xhr.addEventListener('load', onLoad);
xhr.open('GET', '/test');
xhr.send();
const loaded = await prom;
if (!loaded) throw new Error('no reply from server in 5 seconds');
return xhr.responseURL === 'https://tests.wombat.io/test';
});
t.true(result);
});
test('fetch: should rewrite the input argument when it is a string (URL)', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(async () => {
let to;
let response = await Promise.race([
fetch('/test'),
new Promise(resolve => {
to = setTimeout(() => resolve('timed out'), 5000);
})
]);
if (response === 'timed out')
throw new Error('no reply from server in 5 seconds');
clearTimeout(to);
const data = await response.json();
return data.url === '/live/20180803160549mp_/https://tests.wombat.io/test';
});
t.true(result);
});
test('fetch: should rewrite the input argument when it is an Request object', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(async () => {
let to;
let response = await Promise.race([
fetch(
new Request('/test', {
method: 'GET'
})
),
new Promise(resolve => {
to = setTimeout(() => resolve('timed out'), 5000);
})
]);
if (response === 'timed out')
throw new Error('no reply from server in 5 seconds');
clearTimeout(to);
const data = await response.json();
return data.url === '/live/20180803160549mp_/https://tests.wombat.io/test';
});
t.true(result);
});
test('fetch: should rewrite the input argument when it is a object with an href property', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(async () => {
let to;
let response = await Promise.race([
fetch({ href: '/test' }),
new Promise(resolve => {
to = setTimeout(() => resolve('timed out'), 10000);
})
]);
if (response === 'timed out')
throw new Error('no reply from server in 10 seconds');
clearTimeout(to);
const data = await response.json();
return data.url === '/live/20180803160549mp_/https://tests.wombat.io/test';
});
t.true(result);
});
test('Request: should rewrite the input argument to the constructor when it is a string (URL)', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
const req = new Request('/test', { method: 'GET' });
return req.url;
});
t.true(result === mpURL('https://tests.wombat.io/test'));
});
test('Request: should rewrite the input argument to the constructor when it is an object with a url property', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
const req = new Request({ url: '/test' }, { method: 'GET' });
return req.url;
});
t.true(result === mpURL('https://tests.wombat.io/test'));
});

127
wombat/test/overrides-workers.js Normal file
View File

@ -0,0 +1,127 @@
import test from 'ava';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.server = helper.server();
t.context.testPage = helper.testPage();
});
test.afterEach.always(async t => {
if (t.title.includes('SharedWorker')) {
await helper.fullRefresh();
} else {
await helper.ensureSandbox();
}
});
test.after.always(async t => {
await helper.stop();
});
test('Web Workers: should rewrite the URL argument to the constructor of "Worker"', async t => {
const { sandbox, server, testPage } = t.context;
await Promise.all([
new Promise((resolve, reject) => {
const to = setTimeout(
() => reject(new Error('the worker was not started')),
15000
);
testPage.once('workercreated', w => {
clearTimeout(to);
resolve();
});
}),
server.waitForRequest(
'/live/20180803160549wkr_/https://tests.wombat.io/testWorker.js'
),
sandbox.evaluate(() => {
window.theWorker = new Worker('testWorker.js');
})
]);
await sandbox.evaluate(() => {
window.theWorker.terminate();
});
t.pass(
'The worker URL was rewritten when using Worker and is working on the page'
);
});
test('Web Workers: should have a light override applied', async t => {
const { sandbox, server, testPage } = t.context;
const [worker] = await Promise.all([
new Promise((resolve, reject) => {
const to = setTimeout(
() => reject(new Error('the worker was not started')),
15000
);
testPage.once('workercreated', w => {
clearTimeout(to);
resolve(w);
});
}),
server.waitForRequest(
'/live/20180803160549wkr_/https://tests.wombat.io/testWorker.js'
),
sandbox.evaluate(() => {
window.theWorker = new Worker('testWorker.js');
})
]);
const result = await worker
.evaluate(() => ({
fetch: self.isFetchOverridden(),
importScripts: self.isImportScriptOverridden(),
open: self.isAjaxRewritten()
}))
.then(async results => {
await sandbox.evaluate(() => {
window.theWorker.terminate();
});
return results;
});
t.deepEqual(
result,
{ fetch: true, importScripts: true, open: true },
'The light web worker overrides were not applied properly'
);
});
test('Web Workers: should rewrite the URL argument to the constructor of "SharedWorker"', async t => {
const { sandbox, server, testPage } = t.context;
await Promise.all([
server.waitForRequest(
'/live/20180803160549wkr_/https://tests.wombat.io/testWorker.js'
),
sandbox.evaluate(() => {
window.theWorker = new SharedWorker('testWorker.js');
})
]);
t.pass(
'The worker URL was rewritten when using SharedWorker and is working on the page'
);
});
test('Service Worker: should rewrite the URL argument of "navigator.serviceWorker.register"', async t => {
const { sandbox, server, testPage } = t.context;
const result = await sandbox.evaluate(async () => {
const sw = await window.navigator.serviceWorker.register(
'/testServiceWorker.js'
);
await sw.unregister();
return sw.scope;
});
t.true(
result.includes('mp_/https://tests.wombat.io/'),
'rewriting of service workers is not correct'
);
});

557
wombat/test/setup-after-initialization.js Normal file
View File

@ -0,0 +1,557 @@
import test from 'ava';
import TestHelper from './helpers/testHelper';
import * as testedChanges from './helpers/testedValues';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
await helper.initWombat();
});
test.beforeEach(async t => {
t.context.sandbox = helper.sandbox();
t.context.testPage = helper.testPage();
t.context.server = helper.server();
});
test.after.always(async t => {
await helper.stop();
});
test('internal globals: should not have removed _WBWombat from window', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
_WBWombat: {
exists: window._WBWombat != null,
type: typeof window._WBWombat
},
_WBWombatInit: {
exists: window._WBWombatInit != null,
type: typeof window._WBWombatInit
}
})),
{
_WBWombat: { exists: true, type: 'function' },
_WBWombatInit: { exists: true, type: 'function' }
},
'The internal globals _WBWombat and _WBWombatInit are not as expected after initialization'
);
});
test('internal globals: should add the property __WB_replay_top to window that is equal to the same window', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
exists: window.__WB_replay_top != null,
eq: window.__WB_replay_top === window
})),
{
exists: true,
eq: true
},
'The internal global __WB_replay_top is not as expected after initialization'
);
});
test('internal globals: should define the property __WB_top_frame when it is the top replayed page', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
exists: window.__WB_top_frame != null,
neq: window.__WB_top_frame !== window
})),
{
exists: true,
neq: true
},
'The internal global __WB_top_frame is not as expected after initialization'
);
});
test('internal globals: should define the WB_wombat_top property on Object.prototype', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => {
const descriptor = Reflect.getOwnPropertyDescriptor(
Object.prototype,
'WB_wombat_top'
);
if (!descriptor) return { exists: false };
return {
exists: true,
configurable: descriptor.configurable,
enumerable: descriptor.enumerable,
get: typeof descriptor.get,
set: typeof descriptor.set
};
}),
{
exists: true,
configurable: true,
enumerable: false,
get: 'function',
set: 'function'
},
'The property descriptor added to the prototype of Object for WB_wombat_top is not correct'
);
});
test('internal globals: should add the _WB_wombat_location property to window', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
_WB_wombat_location: {
exists: window._WB_wombat_location != null,
type: typeof window._WB_wombat_location
},
WB_wombat_location: {
exists: window.WB_wombat_location != null,
type: typeof window.WB_wombat_location
}
})),
{
_WB_wombat_location: { exists: true, type: 'object' },
WB_wombat_location: { exists: true, type: 'object' }
},
'Wombat location properties on window are not correct'
);
});
test('internal globals: should add the __wb_Date_now property to window', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
exists: window.__wb_Date_now != null,
type: typeof window.__wb_Date_now
})),
{ exists: true, type: 'function' },
'The __wb_Date_now property of window is incorrect'
);
});
test('internal globals: should add the __WB_timediff property to window.Date', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
exists: window.Date.__WB_timediff != null,
type: typeof window.Date.__WB_timediff
})),
{ exists: true, type: 'number' },
'The __WB_timediff property of window.Date is incorrect'
);
});
test('internal globals: should persist the original window.postMessage as __orig_postMessage', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => ({
exists: window.__orig_postMessage != null,
type: typeof window.__orig_postMessage,
isO: window.__orig_postMessage.toString().includes('[native code]')
})),
{ exists: true, type: 'function', isO: true },
'The __WB_timediff property of window.Date is incorrect'
);
});
test('internal globals: should not expose WombatLocation on window', async t => {
const { sandbox, server } = t.context;
t.true(
await sandbox.evaluate(() => !('WombatLocation' in window)),
'WombatLocation should not be exposed directly'
);
});
test('exposed functions - extract_orig: should should extract the original url', async t => {
const { sandbox, server } = t.context;
t.true(
await sandbox.evaluate(
() =>
window._wb_wombat.extract_orig(
'http://localhost:3030/jberlin/sw/20180510171123/https://n0tan3rd.github.io/replay_test/'
) === 'https://n0tan3rd.github.io/replay_test/'
),
'extract_orig could not extract the original URL'
);
});
test('exposed functions - extract_orig: should not modify an un-rewritten url', async t => {
const { sandbox, server } = t.context;
t.true(
await sandbox.evaluate(
() =>
window._wb_wombat.extract_orig(
'https://n0tan3rd.github.io/replay_test/'
) === 'https://n0tan3rd.github.io/replay_test/'
),
'extract_orig modified an original URL'
);
});
test('exposed functions - extract_orig: should be able to extract the original url from an encoded string', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => {
const expected = 'https://n0tan3rd.github.io/replay_test/';
const extractO = window._wb_wombat.extract_orig;
const unicode = extractO(
'\u0068\u0074\u0074\u0070\u003a\u002f\u002f\u006c\u006f\u0063\u0061\u006c\u0068\u006f\u0073\u0074\u003a\u0033\u0030\u0033\u0030\u002f\u006a\u0062\u0065\u0072\u006c\u0069\u006e\u002f\u0073\u0077\u002f\u0032\u0030\u0031\u0038\u0030\u0035\u0031\u0030\u0031\u0037\u0031\u0031\u0032\u0033\u002f\u0068\u0074\u0074\u0070\u0073\u003a\u002f\u002f\u006e\u0030\u0074\u0061\u006e\u0033\u0072\u0064\u002e\u0067\u0069\u0074\u0068\u0075\u0062\u002e\u0069\u006f\u002f\u0072\u0065\u0070\u006c\u0061\u0079\u005f\u0074\u0065\u0073\u0074\u002f'
);
const hex = extractO(
'\x68\x74\x74\x70\x3a\x2f\x2f\x6c\x6f\x63\x61\x6c\x68\x6f\x73\x74\x3a\x33\x30\x33\x30\x2f\x6a\x62\x65\x72\x6c\x69\x6e\x2f\x73\x77\x2f\x32\x30\x31\x38\x30\x35\x31\x30\x31\x37\x31\x31\x32\x33\x2f\x68\x74\x74\x70\x73\x3a\x2f\x2f\x6e\x30\x74\x61\x6e\x33\x72\x64\x2e\x67\x69\x74\x68\x75\x62\x2e\x69\x6f\x2f\x72\x65\x70\x6c\x61\x79\x5f\x74\x65\x73\x74\x2f'
);
return {
unicode:
unicode === expected &&
unicode ===
'\u0068\u0074\u0074\u0070\u0073\u003a\u002f\u002f\u006e\u0030\u0074\u0061\u006e\u0033\u0072\u0064\u002e\u0067\u0069\u0074\u0068\u0075\u0062\u002e\u0069\u006f\u002f\u0072\u0065\u0070\u006c\u0061\u0079\u005f\u0074\u0065\u0073\u0074\u002f',
hex:
hex === expected &&
hex ===
'\x68\x74\x74\x70\x73\x3a\x2f\x2f\x6e\x30\x74\x61\x6e\x33\x72\x64\x2e\x67\x69\x74\x68\x75\x62\x2e\x69\x6f\x2f\x72\x65\x70\x6c\x61\x79\x5f\x74\x65\x73\x74\x2f'
};
}),
{ unicode: true, hex: true },
'extract_orig could not extract the original URL from an encoded string'
);
});
test('exposed functions - rewrite_url: should be able to rewrite an encoded string', async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(() => {
const expected = 'https://n0tan3rd.github.io/replay_test/';
const rewrite_url = window._wb_wombat.rewrite_url;
const unicode = rewrite_url(
'\u0068\u0074\u0074\u0070\u0073\u003a\u002f\u002f\u006e\u0030\u0074\u0061\u006e\u0033\u0072\u0064\u002e\u0067\u0069\u0074\u0068\u0075\u0062\u002e\u0069\u006f\u002f\u0072\u0065\u0070\u006c\u0061\u0079\u005f\u0074\u0065\u0073\u0074\u002f'
);
const hex = rewrite_url(
'\x68\x74\x74\x70\x73\x3a\x2f\x2f\x6e\x30\x74\x61\x6e\x33\x72\x64\x2e\x67\x69\x74\x68\x75\x62\x2e\x69\x6f\x2f\x72\x65\x70\x6c\x61\x79\x5f\x74\x65\x73\x74\x2f'
);
return {
unicode:
unicode ===
`${window.wbinfo.prefix}${window.wbinfo.wombat_ts}${
window.wbinfo.mod
}/${expected}`,
hex:
hex ===
`${window.wbinfo.prefix}${window.wbinfo.wombat_ts}${
window.wbinfo.mod
}/${expected}`
};
}),
{ unicode: true, hex: true },
'rewrite_url could not rewrite an encoded string'
);
});
testedChanges.TestedPropertyDescriptorUpdates.forEach(aTest => {
const msg = 'an property descriptor override should have been applied';
aTest.props.forEach(prop => {
if (aTest.docOrWin) {
test(`${aTest.docOrWin}.${prop}: ${msg}`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
testFn,
aTest.docOrWin,
prop,
aTest.expectedInterface,
aTest.skipGet,
aTest.skipSet
);
t.deepEqual(
result.main,
{ exists: true, good: true },
`The property descriptor for ${aTest.docOrWin}.${prop} is incorrect`
);
for (let i = 0; i < result.sub.length; i++) {
const subTest = result.sub[i];
t.true(subTest.result, subTest.what);
}
});
} else if (aTest.objPaths) {
aTest.objPaths.forEach(objPath => {
test(`${objPath.replace('window.', '')}.${prop}: ${msg}`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
testFn,
objPath,
prop,
aTest.expectedInterface,
aTest.skipGet,
aTest.skipSet
);
t.deepEqual(
result.main,
{ exists: true, good: true },
`The property descriptor for ${objPath.replace(
'window.',
''
)}.${prop} is incorrect`
);
for (let i = 0; i < result.sub.length; i++) {
const subTest = result.sub[i];
t.true(subTest.result, subTest.what);
}
});
});
} else {
test(`${aTest.objPath.replace(
'window.',
''
)}.${prop}: ${msg}`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
testFn,
aTest.objPath,
prop,
aTest.expectedInterface,
aTest.skipGet,
aTest.skipSet
);
t.deepEqual(
result.main,
{ exists: true, good: true },
`The property descriptor for ${aTest.objPath.replace(
'window.',
''
)}.prop is incorrect`
);
for (let i = 0; i < result.sub.length; i++) {
const subTest = result.sub[i];
t.true(subTest.result, subTest.what);
}
});
}
});
function testFn(objectPath, prop, expectedInterface, skipGet, skipSet) {
// get the original and wombat object represented by the object path expression
// eg if objectPath is window.Node.prototype then original === window.Node.prototype and obj === wombatSandbox.window.Node.prototype
const original = window.WombatTestUtil.getOriginalWinDomViaPath(objectPath);
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
objectPath
);
// sometimes we need to skip a property get/set toString check
let skipGetCheck = !!(skipGet && skipGet.indexOf(prop) > -1);
let skipSetCheck = !!(skipSet && skipSet.indexOf(prop) > -1);
// use the reflect object in the wombat sandbox context
const newPD = Reflect.getOwnPropertyDescriptor(existing, prop);
const expectedEntries = Object.entries(expectedInterface);
const tResults = {
main: {
exists: newPD != null,
good: expectedEntries.every(([pk, ptype]) => typeof newPD[pk] === ptype)
},
sub: []
};
const originalPD = window.WombatTestUtil.getOriginalPropertyDescriptorFor(
original,
prop
);
if (originalPD) {
// do a quick deep check first to see if we modified something
tResults.sub.push({
what: 'newPD !== originalPD',
result: expectedEntries.some(
([pk, ptype]) => newPD[pk] !== originalPD[pk]
)
});
// now check each part of the expected property descriptor to make sure nothing went wrong
if (
!skipGetCheck &&
expectedInterface.get &&
newPD.get &&
originalPD.get
) {
tResults.sub.push({
what: `${objectPath}.${prop} getter.toString() !== original getter.toString()`,
result: newPD.get.toString() !== originalPD.get.toString()
});
}
if (
!skipSetCheck &&
expectedInterface.set &&
newPD.set &&
originalPD.set
) {
tResults.sub.push({
what: `${objectPath}.${prop} setter.toString() !== original setter.toString()`,
result: newPD.set.toString() !== originalPD.set.toString()
});
}
if (originalPD.configurable != null && newPD.configurable != null) {
tResults.sub.push({
what: `${objectPath}.${prop} the "new" configurable pd does not equals the original`,
result: newPD.configurable === originalPD.configurable
});
}
if (originalPD.writable != null && newPD.writable != null) {
tResults.sub.push({
what: `${objectPath}.${prop} the "new" writable pd does not equals the original`,
result: newPD.writable === originalPD.writable
});
}
if (originalPD.value != null && newPD.value != null) {
tResults.sub.push({
what: `${objectPath}.${prop} the "new" value pd does equals the original`,
result: newPD.value !== originalPD.value
});
}
}
return tResults;
}
});
testedChanges.TestFunctionChanges.forEach(aTest => {
if (aTest.constructors) {
aTest.constructors.forEach(ctor => {
const niceC = ctor.replace('window.', '');
test(`${niceC}: an constructor override should have been applied`, async t => {
const { sandbox, server } = t.context;
t.true(
await sandbox.evaluate(testFN, ctor),
`The ${ctor} was not updated`
);
function testFN(ctor) {
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
ctor
);
const original = window.WombatTestUtil.getOriginalWinDomViaPath(ctor);
return existing.toString() !== original.toString();
}
});
});
} else if (aTest.objPath && aTest.origs) {
for (let i = 0; i < aTest.fns.length; i++) {
const fn = aTest.fns[i];
const ofn = aTest.origs[i];
const niceWhat = `${aTest.objPath.replace('window.', '')}.${fn}`;
test(`${niceWhat}: an function override should have been applied`, async t => {
const { sandbox, server } = t.context;
t.deepEqual(
await sandbox.evaluate(testFN, aTest.objPath, fn, ofn),
{ ne: true, persisted: true },
`The ${niceWhat} was not updated correctly`
);
function testFN(objPath, fn, ofn) {
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
objPath
);
const original = window.WombatTestUtil.getOriginalWinDomViaPath(
objPath
);
return {
ne: existing[fn].toString() !== original[fn].toString(),
persisted: existing[ofn].toString() === original[fn].toString()
};
}
});
}
} else if (aTest.fnPath) {
test(`${
aTest.fnPath
}: an function override should have been applied`, async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(testFN, aTest.fnPath, aTest.oPath);
t.true(result.ne, `${aTest.fnPath} was not updated`);
if (result.originalPersisted) {
t.true(
result.originalPersisted,
`The persisted original function for ${
aTest.fnPath
} does not match the original`
);
}
function testFN(fnPath, oPath) {
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
fnPath
);
const original = window.WombatTestUtil.getOriginalWinDomViaPath(fnPath);
const result = {
ne: existing.toString() !== original.toString()
};
if (oPath) {
const ofnOnfn = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
oPath
);
result.originalPersisted = ofnOnfn.toString() === original.toString();
}
return result;
}
});
} else if (aTest.fns) {
aTest.fns.forEach(fn => {
const niceWhat = `${aTest.objPath.replace('window.', '')}.${fn}`;
test(`${niceWhat}: an function override should have been applied`, async t => {
const { sandbox, server } = t.context;
const results = await sandbox.evaluate(testFn, aTest.objPath, fn);
if (results.tests) {
results.tests.forEach(({ test, msg }) => {
t.true(test, msg);
});
} else {
t.true(results.test, `${aTest.objPath}.${fn} was not updated`);
}
function testFn(objPath, fn) {
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
objPath
);
const original = window.WombatTestUtil.getOriginalWinDomViaPath(
objPath
);
const result = {};
if (existing[fn] && original[fn]) {
result.test = existing[fn].toString() !== original[fn].toString();
} else if (existing[fn]) {
result.tests = [
{
test: existing[fn] !== null,
msg: `${objPath}.${fn} was not overridden (is undefined/null)`
},
{
test: !existing[fn].toString().includes('[native code]'),
msg: `${objPath}.${fn} was not overridden at all`
}
];
} else {
result.test = false;
}
return result;
}
});
});
} else if (aTest.persisted) {
aTest.persisted.forEach(fn => {
const niceWhat = `${aTest.objPath.replace('window.', '')}.${fn}`;
test(`${niceWhat}: the original function should exist on the overridden object`, async t => {
const { sandbox, server } = t.context;
t.true(
await sandbox.evaluate(testFn, aTest.objPath, fn),
`the original function '${niceWhat}' was not persisted`
);
function testFn(objPath, fn) {
const existing = window.WombatTestUtil.getViaPath(
window.wombatSandbox,
objPath
);
const original = window.WombatTestUtil.getOriginalWinDomViaPath(
objPath
);
return existing[fn].toString() === original[fn].toString();
}
});
});
}
});

147
wombat/test/setup-before-initialization.js Normal file
View File

@ -0,0 +1,147 @@
import test from 'ava';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.before(async t => {
helper = await TestHelper.init(t);
});
test.beforeEach(async t => {
/**
* @type {Frame}
*/
t.context.sandbox = helper.sandbox();
/**
* @type {fastify.FastifyInstance<http2.Http2SecureServer, http2.Http2ServerRequest, http2.Http2ServerResponse>}
*/
t.context.server = helper.server();
});
test.after.always(async t => {
await helper.stop();
});
test('should put _WBWombat on window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
const wbWombat = window._WBWombat;
return { exists: wbWombat != null, type: typeof wbWombat };
});
t.deepEqual(
result,
{ exists: true, type: 'function' },
'_WBWombat should be placed on window before initialization and should be a function'
);
});
test('should not add __WB_replay_top to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.__WB_replay_top == null);
t.true(result, '__WB_replay_top should not exist on window');
});
test('should not add _WB_wombat_location to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window._WB_wombat_location == null
);
t.true(result, '_WB_wombat_location should not exist on window');
});
test('should not add WB_wombat_location to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.WB_wombat_location == null
);
t.true(result, 'WB_wombat_location should not exist on window');
});
test('should not add __WB_check_loc to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.__WB_check_loc == null);
t.true(result, '__WB_check_loc should not exist on window');
});
test('should not add __orig_postMessage property on window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.__orig_postMessage == null
);
t.true(result, '__orig_postMessage should not exist on window');
});
test('should not add __WB_top_frame to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.__WB_top_frame == null);
t.true(result, '__WB_top_frame should not exist on window');
});
test('should not add __wb_Date_now to window', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.__wb_Date_now == null);
t.true(result, '__wb_Date_now should not exist on window');
});
test('should not expose CustomStorage', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() =>
window.Storage.toString().includes('[native code]')
);
t.true(result, 'CustomStorage should not exist on window');
});
test('should not expose FuncMap', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.FuncMap == null);
t.true(result, 'FuncMap should not exist on window');
});
test('should not expose SameOriginListener', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => window.SameOriginListener == null
);
t.true(result, 'SameOriginListener should not exist on window');
});
test('should not expose WrappedListener', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => window.WrappedListener == null);
t.true(result, 'WrappedListener should not exist on window');
});
test('should not add the __WB_pmw property to Object.prototype', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() => Object.prototype.__WB_pmw == null
);
t.true(result, 'Object.prototype.__WB_pmw should be undefined');
});
test('should not add the WB_wombat_top property to Object.prototype', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(
() =>
Object.prototype.WB_wombat_top == null &&
!Object.hasOwnProperty('WB_wombat_top')
);
t.true(result, 'Object.prototype.WB_wombat_top should be undefined');
});
test('should not have patched Element.prototype.insertAdjacentHTML', async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() =>
window.Element.prototype.insertAdjacentHTML
.toString()
.includes('[native code]')
);
t.true(
result,
'Element.prototype.insertAdjacentHTML should not have been patched'
);
});

132
wombat/test/setup-initialization.js Normal file
View File

@ -0,0 +1,132 @@
import test from 'ava';
import TestHelper from './helpers/testHelper';
/**
* @type {TestHelper}
*/
let helper = null;
test.serial.before(async t => {
helper = await TestHelper.init(t);
});
test.serial.beforeEach(async t => {
/**
* @type {Frame}
*/
t.context.sandbox = helper.sandbox();
/**
* @type {fastify.FastifyInstance<http2.Http2SecureServer, http2.Http2ServerRequest, http2.Http2ServerResponse>}
*/
t.context.server = helper.server();
});
test.serial.afterEach.always(async t => {
await helper.cleanup();
});
test.serial.after.always(async t => {
await helper.stop();
});
test.serial(
'should not be possible using the function Wombat a constructor',
async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
try {
new window.Wombat(window, window.wbinfo);
return false;
} catch (e) {
return true;
}
});
t.true(result, 'Wombat can be used as an constructor');
}
);
test.serial(
'should not be possible by invoking the function Wombat',
async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
try {
window.Wombat(window, window.wbinfo);
return false;
} catch (e) {
return true;
}
});
t.true(result, 'Wombat can be created via invoking the function Wombat');
}
);
test.serial(
'using _WBWombatInit as a plain function: should not throw an error',
async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
try {
window._WBWombatInit(window.wbinfo);
return window._wb_wombat != null;
} catch (e) {
return false;
}
});
t.true(result, 'Wombat can not be initialized using _WBWombatInit');
}
);
test.serial(
'using _WBWombatInit as a plain function: should not return an object containing the exposed functions',
async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
try {
return (
window._WBWombatInit(window.wbinfo) == null &&
window._wb_wombat != null
);
} catch (e) {
return false;
}
});
t.true(
result,
'window._WBWombatInit(window.wbinfo) should not return anything'
);
}
);
test.serial(
'using _WBWombatInit as a plain function: should add the property _wb_wombat to the window which is an object containing the exposed functions',
async t => {
const { sandbox, server } = t.context;
const result = await sandbox.evaluate(() => {
window._WBWombatInit(window.wbinfo);
return {
actual: window._wb_wombat.actual,
extract_orig: typeof window._wb_wombat.extract_orig,
rewrite_url: typeof window._wb_wombat.rewrite_url,
watch_elem: typeof window._wb_wombat.watch_elem,
init_new_window_wombat: typeof window._wb_wombat.init_new_window_wombat,
init_paths: typeof window._wb_wombat.init_paths,
local_init: typeof window._wb_wombat.local_init
};
});
t.deepEqual(
result,
{
actual: true,
extract_orig: 'function',
rewrite_url: 'function',
watch_elem: 'function',
init_new_window_wombat: 'function',
init_paths: 'function',
local_init: 'function'
},
`window._wb_wombat does not have the expected interface`
);
}
);

4989
wombat/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff