backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

enabled auto-fetching of video, audio resources in wombat in non-proxy mode and proxy mode (#427)

Browse Source
This commit is contained in:
John Berlin 2018-12-05 19:03:00 -05:00 committed by Ilya Kreymer
parent 3235c382a5
commit 323edcf47c
4 changed files with 581 additions and 384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

323
pywb/static/autoFetchWorker.js
View File

@ -3,6 +3,12 @@
var STYLE_REGEX = /(url\s*\(\s*[\\"']*)([^)'"]+)([\\"']*\s*\))/gi; var STYLE_REGEX = /(url\s*\(\s*[\\"']*)([^)'"]+)([\\"']*\s*\))/gi;
var IMPORT_REGEX = /(@import\s+[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi; var IMPORT_REGEX = /(@import\s+[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi;
var srcsetSplit = /\s*(\S*\s+[\d.]+[wx]),|(?:\s*,(?:\s+|(?=https?:)))/; var srcsetSplit = /\s*(\S*\s+[\d.]+[wx]),|(?:\s*,(?:\s+|(?=https?:)))/;
var DefaultNumImFetches = 30;
var FullImgQDrainLen = 10;
var DefaultNumAvFetches = 5;
var FullAVQDrainLen = 5;
var DataURLPrefix = 'data:';
// the autofetcher instance for this worker // the autofetcher instance for this worker
var autofetcher = null; var autofetcher = null;
@ -41,7 +47,7 @@ self.onmessage = function (event) {
var data = event.data; var data = event.data;
switch (data.type) { switch (data.type) {
case 'values': case 'values':
autofetcher.autofetchMediaSrcset(data); autofetcher.autoFetch(data);
break; break;
} }
}; };
@ -53,53 +59,24 @@ function AutoFetcher(init) {
this.prefix = init.prefix; this.prefix = init.prefix;
this.mod = init.mod; this.mod = init.mod;
this.prefixMod = init.prefix + init.mod; this.prefixMod = init.prefix + init.mod;
this.rwRe = new RegExp(init.rwRe);
// relative url, WorkerLocation is set by owning document // relative url, WorkerLocation is set by owning document
this.relative = init.prefix.split(location.origin)[1]; this.relative = init.prefix.split(location.origin)[1];
// schemeless url // schemeless url
this.schemeless = '/' + this.relative; this.schemeless = '/' + this.relative;
// local cache of URLs fetched, to reduce server load // local cache of URLs fetched, to reduce server load
this.seen = {}; this.seen = {};
// array of URL to be fetched // array of URLs to be fetched
this.queue = []; this.queue = [];
this.avQueue = [];
// should we queue a URL or not // should we queue a URL or not
this.queuing = false; this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this); this.urlExtractor = this.urlExtractor.bind(this);
this.fetchDone = this.fetchDone.bind(this); this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
} }
AutoFetcher.prototype.fixupURL = function (url) {
// attempt to fix up the url and do our best to ensure we can get dat 200 OK!
if (url.indexOf(this.prefixMod) === 0) {
return url;
}
if (url.indexOf(this.relative) === 0) {
return url.replace(this.relative, this.prefix);
}
if (url.indexOf(this.schemeless) === 0) {
return url.replace(this.schemeless, this.prefix);
}
if (url.indexOf(this.prefix) !== 0) {
return this.prefix + url;
}
return url;
};
AutoFetcher.prototype.queueURL = function (url) {
// ensure we do not request data urls
if (url.indexOf('data:') === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.queue.push(url);
};
AutoFetcher.prototype.urlExtractor = function (match, n1, n2, n3, offset, string) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
this.queueURL(this.fixupURL(n2));
return n1 + n2 + n3;
};
AutoFetcher.prototype.delay = function () { AutoFetcher.prototype.delay = function () {
// 2 second delay seem reasonable // 2 second delay seem reasonable
return new Promise(function (resolve, reject) { return new Promise(function (resolve, reject) {
@ -107,47 +84,105 @@ AutoFetcher.prototype.delay = function () {
}); });
}; };
AutoFetcher.prototype.fetchDone = function () { AutoFetcher.prototype.imgFetchDone = function () {
this.queuing = false;
if (this.queue.length > 0) { if (this.queue.length > 0) {
// we have a Q of some length drain it // we have a Q of some length drain it
var autofetcher = this; var autofetcher = this;
this.delay().then(function () { this.delay().then(function () {
autofetcher.fetchAll(); autofetcher.queuing = false;
autofetcher.fetchImgs();
}); });
} else {
this.queuing = false;
} }
}; };
AutoFetcher.prototype.fetchAll = function () { AutoFetcher.prototype.avFetchDone = function () {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
}
};
AutoFetcher.prototype.fetchAV = function () {
if (this.queuingAV || this.avQueue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (this.avQueue.length > 0 && runningFetchers.length <= DefaultNumAvFetches) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
};
AutoFetcher.prototype.fetchImgs = function () {
if (this.queuing || this.queue.length === 0) { if (this.queuing || this.queue.length === 0) {
return; return;
} }
// the number of fetches is limited to a maximum of 60 outstanding fetches // the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is 50 but if the size(queue) <= 10 // the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch // we add them to the current batch
this.queuing = true; this.queuing = true;
var runningFetchers = []; var runningFetchers = [];
while (this.queue.length > 0 && runningFetchers.length <= 50) { while (this.queue.length > 0 && runningFetchers.length <= DefaultNumImFetches) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop)) runningFetchers.push(fetch(this.queue.shift()).catch(noop))
} }
if (this.queue.length <= 10) { if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) { while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop)) runningFetchers.push(fetch(this.queue.shift()).catch(noop))
} }
} }
Promise.all(runningFetchers) Promise.all(runningFetchers)
.then(this.fetchDone) .then(this.imgFetchDone)
.catch(this.fetchDone); .catch(this.imgFetchDone);
}; };
AutoFetcher.prototype.extractMedia = function (mediaRules) { AutoFetcher.prototype.queueNonAVURL = function (url) {
// this is a broken down rewrite_style // ensure we do not request data urls
if (mediaRules == null || mediaRules.values === null) return; if (url.indexOf(DataURLPrefix) === 0) return;
var rules = mediaRules.values; // check to see if we have seen this url before in order
for (var i = 0; i < rules.length; i++) { // to lessen the load against the server content is fetched from
var rule = rules[i]; if (this.seen[url] != null) return;
rule.replace(STYLE_REGEX, this.urlExtractor) this.seen[url] = true;
.replace(IMPORT_REGEX, this.urlExtractor); this.queue.push(url);
};
AutoFetcher.prototype.queueAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.maybeResolveURL = function (url, base) {
// given a url and base url returns a resolved full URL or
// null if resolution was unsuccessful
try {
var _url = new URL(url, base);
return _url.href;
} catch (e) {
return null;
} }
}; };
@ -163,85 +198,149 @@ AutoFetcher.prototype.maybeFixUpRelSchemelessPrefix = function (url) {
return null; return null;
}; };
AutoFetcher.prototype.maybeResolveURL = function (url, base) { AutoFetcher.prototype.maybeFixUpURL = function (url, resolveOpts) {
// given a url and base url returns a resolved full URL or // attempt to fix up the url and do our best to ensure we can get dat 200 OK!
// null if resolution was unsuccessful if (this.rwRe.test(url)) {
try { return url;
var _url = new URL(url, base); }
return _url.href; var mod = resolveOpts.mod || 'mp_';
} catch (e) { // first check for / (relative) or // (schemeless) rewritten urls
return null; var maybeFixed = this.maybeFixUpRelSchemelessPrefix(url);
if (maybeFixed != null) {
return maybeFixed;
}
// resolve URL against tag src
if (resolveOpts.tagSrc != null) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.tagSrc);
if (maybeFixed != null) {
return this.prefix + mod + '/' + maybeFixed;
}
}
// finally last attempt resolve the originating documents base URI
if (resolveOpts.docBaseURI) {
maybeFixed = this.maybeResolveURL(url, resolveOpts.docBaseURI);
if (maybeFixed != null) {
return this.prefix + mod + '/' + maybeFixed;
}
}
// not much to do now.....
return this.prefixMod + '/' + url;
};
AutoFetcher.prototype.urlExtractor = function (match, n1, n2, n3, offset, string) {
// Same function as style_replacer in wombat.rewrite_style, n2 is our URL
this.queueNonAVURL(n2);
return n1 + n2 + n3;
};
AutoFetcher.prototype.handleMedia = function (mediaRules) {
// this is a broken down rewrite_style
if (mediaRules == null || mediaRules.length === 0) return;
// var rules = mediaRules.values;
for (var i = 0; i < mediaRules.length; i++) {
mediaRules[i]
.replace(STYLE_REGEX, this.urlExtractor)
.replace(IMPORT_REGEX, this.urlExtractor);
} }
}; };
AutoFetcher.prototype.handleSrc = function (srcValues, context) {
AutoFetcher.prototype.fixupURLSrcSet = function (url, tagSrc, context) { var resolveOpts = { 'docBaseURI': context.docBaseURI };
// attempt to fix up the url and do our best to ensure we can get dat 200 OK! if (srcValues.value) {
if (url.indexOf(this.prefix) !== 0) { resolveOpts.mod = srcValues.mod;
// first check for / (relative) or // (schemeless) rewritten urls if (resolveOpts.mod === 1) {
var maybeFixed = this.maybeFixUpRelSchemelessPrefix(url); return this.queueNonAVURL(this.maybeFixUpURL(srcValues.value.trim(), resolveOpts));
if (maybeFixed != null) {
return maybeFixed;
} }
// resolve URL against tag src return this.queueAVURL(this.maybeFixUpURL(srcValues.value.trim(), resolveOpts));
if (tagSrc != null) { }
maybeFixed = this.maybeResolveURL(url, tagSrc); var len = srcValues.values.length;
if (maybeFixed != null) { for (var i = 0; i < len; i++) {
return this.prefix + 'im_/' + maybeFixed; var value = srcValues.values[i];
resolveOpts.mod = value.mod;
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(this.maybeFixUpURL(value.src, resolveOpts));
} else {
this.queueAVURL(this.maybeFixUpURL(value.src, resolveOpts));
}
}
};
AutoFetcher.prototype.extractSrcSetNotPreSplit = function (ssV, resolveOpts) {
// was from extract from local doc so we need to duplicate work
var srcsetValues = ssV.split(srcsetSplit);
for (var i = 0; i < srcsetValues.length; i++) {
// grab the URL not width/height key
if (srcsetValues[i]) {
var value = srcsetValues[i].trim().split(' ')[0];
var maybeResolvedURL = this.maybeFixUpURL(value.trim(), resolveOpts);
if (resolveOpts.mod === 'im_') {
this.queueNonAVURL(maybeResolvedURL);
} else {
this.queueAVURL(maybeResolvedURL);
} }
} }
// finally last attempt resolve the originating documents base URI
maybeFixed = this.maybeResolveURL(url, context.docBaseURI);
if (maybeFixed != null) {
return this.prefix + 'im_/' + maybeFixed;
}
// not much to do now.....
return this.prefixMod + '/' + url;
} }
return url;
}; };
AutoFetcher.prototype.extractSrcset = function (srcsets, context) { AutoFetcher.prototype.extractSrcset = function (srcsets, context) {
if (srcsets == null || srcsets.values == null) return; // was rewrite_srcset and only need to q
var srcsetValues = srcsets.values; for (var i = 0; i < srcsets.length; i++) {
if (!srcsets.presplit) {
// was from extract from local doc so we need to duplicate work
return this.srcsetNotPreSplit(srcsetValues, context);
}
// was rewrite_srcset so just ensure we just
for (var i = 0; i < srcsetValues.length; i++) {
// grab the URL not width/height key // grab the URL not width/height key
this.queueURL(srcsetValues[i].split(' ')[0]); var url = srcsets[i].split(' ')[0];
} if (context.mod === 'im_') {
}; this.queueNonAVURL(url);
} else {
AutoFetcher.prototype.srcsetNotPreSplit = function (values, context) { this.queueAVURL(url);
// was from extract from local doc so we need to duplicate work
var j;
for (var i = 0; i < values.length; i++) {
var srcsetValues = values[i].srcset.split(srcsetSplit);
var tagSrc = values[i].tagSrc;
for (j = 0; j < srcsetValues.length; j++) {
// grab the URL not width/height key
if (Boolean(srcsetValues[j])) {
var value = srcsetValues[j].trim().split(' ')[0];
this.queueURL(this.fixupURLSrcSet(value, tagSrc, context));
}
} }
} }
}; };
AutoFetcher.prototype.autofetchMediaSrcset = function (data) { AutoFetcher.prototype.handleSrcset = function (srcset, context) {
var resolveOpts = { 'docBaseURI': context.docBaseURI };
if (srcset.value) {
// we have a single value, this srcset came from either
// preserveDataSrcset (not presplit) preserveSrcset (presplit)
resolveOpts.mod = srcset.mod;
if (!srcset.presplit) {
// extract URLs from the srcset string
return this.extractSrcSetNotPreSplit(srcset.value, resolveOpts);
}
// we have an array of srcset URL strings
return this.extractSrcset(srcset.value, resolveOpts);
}
// we have an array of values, these srcsets came from extractFromLocalDoc
var len = srcset.values.length;
for (var i = 0; i < len; i++) {
var ssv = srcset.values[i];
resolveOpts.mod = ssv.mod;
resolveOpts.tagSrc = ssv.tagSrc;
this.extractSrcSetNotPreSplit(ssv.srcset, resolveOpts);
}
};
AutoFetcher.prototype.autoFetch = function (data) {
// we got a message and now we autofetch! // we got a message and now we autofetch!
// these calls turn into no ops if they have no work // these calls turn into no ops if they have no work
this.extractMedia(data.media); if (data.media) {
this.extractSrcset(data.srcset, data.context); this.handleMedia(data.media);
this.fetchAll(); }
if (data.src) {
this.handleSrc(data.src, data.context || {});
}
if (data.srcset) {
this.handleSrcset(data.srcset, data.context || {});
}
this.fetchImgs();
this.fetchAV();
}; };
// initialize ourselves from the query params :) // initialize ourselves from the query params :)
try { try {
var loc = new self.URL(location); var loc = new self.URL(location.href);
autofetcher = new AutoFetcher(JSON.parse(loc.searchParams.get('init'))); autofetcher = new AutoFetcher(JSON.parse(loc.searchParams.get('init')));
} catch (e) { } catch (e) {
// likely we are in an older version of safari // likely we are in an older version of safari

183
pywb/static/autoFetchWorkerProxyMode.js
View File

@ -3,6 +3,11 @@
var STYLE_REGEX = /(url\s*\(\s*[\\"']*)([^)'"]+)([\\"']*\s*\))/gi; var STYLE_REGEX = /(url\s*\(\s*[\\"']*)([^)'"]+)([\\"']*\s*\))/gi;
var IMPORT_REGEX = /(@import\s+[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi; var IMPORT_REGEX = /(@import\s+[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi;
var srcsetSplit = /\s*(\S*\s+[\d.]+[wx]),|(?:\s*,(?:\s+|(?=https?:)))/; var srcsetSplit = /\s*(\S*\s+[\d.]+[wx]),|(?:\s*,(?:\s+|(?=https?:)))/;
var DefaultNumImFetches = 30;
var FullImgQDrainLen = 10;
var DefaultNumAvFetches = 5;
var FullAVQDrainLen = 5;
var DataURLPrefix = 'data:';
// the autofetcher instance for this worker // the autofetcher instance for this worker
var autofetcher = null; var autofetcher = null;
@ -53,26 +58,119 @@ function AutoFetcher() {
} }
// local cache of URLs fetched, to reduce server load // local cache of URLs fetched, to reduce server load
this.seen = {}; this.seen = {};
// array of URL to be fetched // array of URLs to be fetched
this.queue = []; this.queue = [];
this.avQueue = [];
// should we queue a URL or not // should we queue a URL or not
this.queuing = false; this.queuing = false;
// a URL to resolve relative URLs found in the cssText of CSSMedia rules. // a URL to resolve relative URLs found in the cssText of CSSMedia rules.
this.currentResolver = null; this.currentResolver = null;
// should we queue a URL or not
this.queuing = false;
this.queuingAV = false;
this.urlExtractor = this.urlExtractor.bind(this); this.urlExtractor = this.urlExtractor.bind(this);
this.fetchDone = this.fetchDone.bind(this); this.imgFetchDone = this.imgFetchDone.bind(this);
this.avFetchDone = this.avFetchDone.bind(this);
} }
AutoFetcher.prototype.queueURL = function (url) { AutoFetcher.prototype.delay = function () {
// 2 second delay seem reasonable
return new Promise(function (resolve, reject) {
setTimeout(resolve, 2000);
});
};
AutoFetcher.prototype.imgFetchDone = function () {
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuing = false;
autofetcher.fetchImgs();
});
} else {
this.queuing = false;
}
};
AutoFetcher.prototype.avFetchDone = function () {
if (this.avQueue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
this.delay().then(function () {
autofetcher.queuingAV = false;
autofetcher.fetchAV();
});
} else {
this.queuingAV = false;
}
};
AutoFetcher.prototype.fetchAV = function () {
if (this.queuingAV || this.avQueue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumAvFetches + FullAVQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumAvFetches but if the size(avQueue) <= FullAVQDrainLen
// we add them to the current batch. Because audio video resources might be big
// we limit how many we fetch at a time drastically
this.queuingAV = true;
var runningFetchers = [];
while (this.avQueue.length > 0 && runningFetchers.length <= DefaultNumAvFetches) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
if (this.avQueue.length <= FullAVQDrainLen) {
while (this.avQueue.length > 0) {
runningFetchers.push(fetch(this.avQueue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.avFetchDone)
.catch(this.avFetchDone);
};
AutoFetcher.prototype.fetchImgs = function () {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of DefaultNumImFetches + FullImgQDrainLen outstanding fetches
// the baseline maximum number of fetches is DefaultNumImFetches but if the size(queue) <= FullImgQDrainLen
// we add them to the current batch
this.queuing = true;
var runningFetchers = [];
while (this.queue.length > 0 && runningFetchers.length <= DefaultNumImFetches) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
if (this.queue.length <= FullImgQDrainLen) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.imgFetchDone)
.catch(this.imgFetchDone);
};
AutoFetcher.prototype.queueNonAVURL = function (url) {
// ensure we do not request data urls // ensure we do not request data urls
if (url.indexOf('data:') === 0) return; if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order // check to see if we have seen this url before in order
// to lessen the load against the server content is autofetchd from // to lessen the load against the server content is fetched from
if (this.seen[url] != null) return; if (this.seen[url] != null) return;
this.seen[url] = true; this.seen[url] = true;
this.queue.push(url); this.queue.push(url);
}; };
AutoFetcher.prototype.queueAVURL = function (url) {
// ensure we do not request data urls
if (url.indexOf(DataURLPrefix) === 0) return;
// check to see if we have seen this url before in order
// to lessen the load against the server content is fetched from
if (this.seen[url] != null) return;
this.seen[url] = true;
this.avQueue.push(url);
};
AutoFetcher.prototype.safeResolve = function (url, resolver) { AutoFetcher.prototype.safeResolve = function (url, resolver) {
// Guard against the exception thrown by the URL constructor if the URL or resolver is bad // Guard against the exception thrown by the URL constructor if the URL or resolver is bad
// if resolver is undefined/null then this function passes url through // if resolver is undefined/null then this function passes url through
@ -95,52 +193,11 @@ AutoFetcher.prototype.urlExtractor = function (match, n1, n2, n3, offset, string
// (resolvedURL will be undefined if an error occurred) // (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(n2, this.currentResolver); var resolvedURL = this.safeResolve(n2, this.currentResolver);
if (resolvedURL) { if (resolvedURL) {
this.queueURL(resolvedURL); this.queueNonAVURL(resolvedURL);
} }
return n1 + n2 + n3; return n1 + n2 + n3;
}; };
AutoFetcher.prototype.delay = function () {
// 2 second delay seem reasonable
return new Promise(function (resolve, reject) {
setTimeout(resolve, 2000);
});
};
AutoFetcher.prototype.fetchDone = function () {
this.queuing = false;
if (this.queue.length > 0) {
// we have a Q of some length drain it
var autofetcher = this;
// wait 2 seconds before doing another batch
this.delay().then(function () {
autofetcher.fetchAll();
});
}
};
AutoFetcher.prototype.fetchAll = function () {
if (this.queuing || this.queue.length === 0) {
return;
}
// the number of fetches is limited to a maximum of 60 outstanding fetches
// the baseline maximum number of fetches is 50 but if the size(queue) <= 10
// we add them to the current batch this.queuing = true;
this.queuing = true;
var runningFetchers = [];
while (this.queue.length > 0 && runningFetchers.length <= 50) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
if (this.queue.length <= 10) {
while (this.queue.length > 0) {
runningFetchers.push(fetch(this.queue.shift()).catch(noop))
}
}
Promise.all(runningFetchers)
.then(this.fetchDone)
.catch(this.fetchDone);
};
AutoFetcher.prototype.extractMedia = function (mediaRules) { AutoFetcher.prototype.extractMedia = function (mediaRules) {
// this is a broken down rewrite_style // this is a broken down rewrite_style
if (mediaRules == null) return; if (mediaRules == null) return;
@ -165,13 +222,17 @@ AutoFetcher.prototype.extractSrcset = function (srcsets) {
extractedSrcSet = srcsets[i]; extractedSrcSet = srcsets[i];
ssSplit = extractedSrcSet.srcset.split(srcsetSplit); ssSplit = extractedSrcSet.srcset.split(srcsetSplit);
for (j = 0; j < ssSplit.length; j++) { for (j = 0; j < ssSplit.length; j++) {
if (Boolean(ssSplit[j])) { if (ssSplit[j]) {
srcsetValue = ssSplit[j].trim(); srcsetValue = ssSplit[j].trim();
if (srcsetValue.length > 0) { if (srcsetValue.length > 0) {
// resolve the URL in an exceptionless manner (resolvedURL will be undefined if an error occurred) // resolve the URL in an exceptionless manner (resolvedURL will be undefined if an error occurred)
var resolvedURL = this.safeResolve(srcsetValue.split(' ')[0], extractedSrcSet.resolve); var resolvedURL = this.safeResolve(srcsetValue.split(' ')[0], extractedSrcSet.resolve);
if (resolvedURL) { if (resolvedURL) {
this.queueURL(resolvedURL); if (extractedSrcSet.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
} else {
this.queueAVURL(resolvedURL);
}
} }
} }
} }
@ -179,12 +240,34 @@ AutoFetcher.prototype.extractSrcset = function (srcsets) {
} }
}; };
AutoFetcher.prototype.extractSrc = function (srcVals) {
// preservation worker in proxy mode sends us the value of the srcset attribute of an element
// and a URL to correctly resolve relative URLS. Thus we must recreate rewrite_srcset logic here
if (srcVals == null || srcVals.length === 0) return;
var length = srcVals.length;
var srcVal;
for (var i = 0; i < length; i++) {
srcVal = srcVals[i];
var resolvedURL = this.safeResolve(srcVal.src, srcVal.resolve);
if (resolvedURL) {
if (srcVal.mod === 'im_') {
this.queueNonAVURL(resolvedURL);
} else {
this.queueAVURL(resolvedURL);
}
}
}
};
AutoFetcher.prototype.autofetchMediaSrcset = function (data) { AutoFetcher.prototype.autofetchMediaSrcset = function (data) {
// we got a message and now we autofetch! // we got a message and now we autofetch!
// these calls turn into no ops if they have no work // these calls turn into no ops if they have no work
this.extractMedia(data.media); this.extractMedia(data.media);
this.extractSrcset(data.srcset); this.extractSrcset(data.srcset);
this.fetchAll(); this.extractSrc(data.src);
this.fetchImgs();
this.fetchAV();
}; };
autofetcher = new AutoFetcher(); autofetcher = new AutoFetcher();

254
pywb/static/wombat.js
View File

@ -186,13 +186,30 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
} }
function isImageSrcset(elem) { function isSavedSrcSrcset(elem) {
if (elem.tagName === 'IMG') return true; // returns true or false to indicate if the supplied element may have attributes that are auto-fetched
return elem.tagName === 'SOURCE' && elem.parentElement && elem.parentElement.tagName === 'PICTURE'; switch (elem.tagName) {
case 'IMG':
case 'VIDEO':
case 'AUDIO':
return true;
case 'SOURCE':
if (!elem.parentElement) return false;
switch (elem.parentElement.tagName) {
case 'PICTURE':
case 'VIDEO':
case 'AUDIO':
return true;
default:
return false;
}
default:
return false;
}
} }
function isImageDataSrcset(elem) { function isSavedDataSrcSrcset(elem) {
if (isImageSrcset(elem)) return elem.dataset.srcset != null; if (elem.dataset.srcset != null) return isSavedSrcSrcset(elem);
return false; return false;
} }
@ -1162,7 +1179,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
} else if (lowername == "style") { } else if (lowername == "style") {
value = rewrite_style(value); value = rewrite_style(value);
} else if (lowername == "srcset") { } else if (lowername == "srcset") {
value = rewrite_srcset(value, isImageSrcset(this)); value = rewrite_srcset(value, this);
} }
} }
orig_setAttribute.call(this, name, value); orig_setAttribute.call(this, name, value);
@ -1347,25 +1364,35 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
//============================================ //============================================
function initAutoFetchWorker() { function initAutoFetchWorker(rwRe) {
if (!wbUseAFWorker) { if (!wbUseAFWorker) {
return; return;
} }
var isTop = $wbwindow === $wbwindow.__WB_replay_top; var isTop = $wbwindow === $wbwindow.__WB_replay_top;
function AutoFetchWorker(prefix, mod) { function AutoFetchWorker(opts) {
if (!(this instanceof AutoFetchWorker)) { if (!(this instanceof AutoFetchWorker)) {
return new AutoFetchWorker(prefix, mod); return new AutoFetchWorker(opts);
} }
this.checkIntervalCB = this.checkIntervalCB.bind(this); // specifically target the elements we desire
this.elemSelector = ['img', 'source', 'video', 'audio'].map(function (which) {
if (which === 'source') {
return ['picture > ', 'video > ', 'audio >'].map(function (parent) {
return parent + which + '[srcset], ' + parent + which + '[data-srcset], ' + parent + which + '[data-src]'
}).join(', ');
} else {
return which + '[srcset], ' + which + '[data-srcset], ' + which + '[data-src]';
}
}).join(', ');
if (isTop) { if (isTop) {
// we are top and can will own this worker // we are top and can will own this worker
// setup URL for the kewl case // setup URL for the kewl case
// Normal replay and preservation mode pworker setup, its all one origin so YAY! // Normal replay and preservation mode pworker setup, its all one origin so YAY!
var workerURL = wbinfo.static_prefix + var workerURL = (wbinfo.auto_fetch_worker_prefix || wbinfo.static_prefix) +
'autoFetchWorker.js?init='+ 'autoFetchWorker.js?init='+
encodeURIComponent(JSON.stringify({ 'mod': mod, 'prefix': prefix })); encodeURIComponent(JSON.stringify(opts));
this.worker = new $wbwindow.Worker(workerURL); this.worker = new $wbwindow.Worker(workerURL);
} else { } else {
// add only the portions of the worker interface we use since we are not top and if in proxy mode start check polling // add only the portions of the worker interface we use since we are not top and if in proxy mode start check polling
@ -1381,20 +1408,17 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
} }
AutoFetchWorker.prototype.checkIntervalCB = function () {
this.extractFromLocalDoc();
};
AutoFetchWorker.prototype.deferredSheetExtraction = function (sheet) { AutoFetchWorker.prototype.deferredSheetExtraction = function (sheet) {
var rules = sheet.cssRules || sheet.rules; var rules = sheet.cssRules || sheet.rules;
// if no rules this a no op // if no rules this a no op
if (!rules || rules.length === 0) return; if (!rules || rules.length === 0) return;
var self = this; var afw = this;
function extract() { // defer things until next time the Promise.resolve Qs are cleared
$wbwindow.Promise.resolve().then(function () {
// loop through each rule of the stylesheet // loop through each rule of the stylesheet
var media = []; var media = [];
for (var j = 0; j < rules.length; ++j) { for (var i = 0; i < rules.length; ++i) {
var rule = rules[j]; var rule = rules[i];
if (rule.type === CSSRule.MEDIA_RULE) { if (rule.type === CSSRule.MEDIA_RULE) {
// we are a media rule so get its text // we are a media rule so get its text
media.push(rule.cssText); media.push(rule.cssText);
@ -1402,11 +1426,9 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
if (media.length > 0) { if (media.length > 0) {
// we have some media rules to preserve // we have some media rules to preserve
self.preserveMedia(media); afw.preserveMedia(media);
} }
} });
// defer things until next time the Promise.resolve Qs are cleared
$wbwindow.Promise.resolve().then(extract);
}; };
AutoFetchWorker.prototype.terminate = function () { AutoFetchWorker.prototype.terminate = function () {
@ -1416,29 +1438,29 @@ var _WBWombat = function($wbwindow, wbinfo) {
AutoFetchWorker.prototype.postMessage = function (msg, deferred) { AutoFetchWorker.prototype.postMessage = function (msg, deferred) {
if (deferred) { if (deferred) {
var self = this; var afw = this;
return Promise.resolve().then(function () { return Promise.resolve().then(function () {
self.worker.postMessage(msg); afw.worker.postMessage(msg);
}); });
} }
this.worker.postMessage(msg); this.worker.postMessage(msg);
}; };
AutoFetchWorker.prototype.preserveSrcset = function (srcset) { AutoFetchWorker.prototype.preserveSrcset = function (srcset, mod) {
// send values from rewrite_srcset to the worker deferred // send values from rewrite_srcset to the worker deferred
// to ensure the page viewer sees the images first // to ensure the page viewer sees the images first
this.postMessage({ this.postMessage({
'type': 'values', 'type': 'values',
'srcset': {'values': srcset, 'presplit': true}, 'srcset': { 'value': srcset, 'mod': mod, 'presplit': true },
}, true); }, true);
}; };
AutoFetchWorker.prototype.preserveDataSrcset = function (srcset) { AutoFetchWorker.prototype.preserveDataSrcset = function (elem) {
// send values from rewrite_attr srcset to the worker deferred // send values from rewrite_attr srcset to the worker deferred
// to ensure the page viewer sees the images first // to ensure the page viewer sees the images first
this.postMessage({ this.postMessage({
'type': 'values', 'type': 'values',
'srcset': {'values': srcset, 'presplit': false}, 'srcset': {'value': elem.dataset.srcset, 'mod': this.rwMod(elem), 'presplit': false},
}, true); }, true);
}; };
@ -1447,91 +1469,86 @@ var _WBWombat = function($wbwindow, wbinfo) {
this.postMessage({'type': 'values', 'media': media}, true); this.postMessage({'type': 'values', 'media': media}, true);
}; };
AutoFetchWorker.prototype.extractSrcset = function (elem) { AutoFetchWorker.prototype.getSrcset = function (elem) {
if (wb_getAttribute) { if (wb_getAttribute) {
return wb_getAttribute.call(elem, 'srcset'); return wb_getAttribute.call(elem, 'srcset');
} }
return elem.getAttribute('srcset'); return elem.getAttribute('srcset');
}; };
AutoFetchWorker.prototype.checkForPictureSourceDataSrcsets = function () { AutoFetchWorker.prototype.rwMod = function (elem) {
var dataSS = $wbwindow.document.querySelectorAll('img[data-srcset], source[data-srcset]'); return elem.tagName === "SOURCE" ?
var elem; elem.parentElement.tagName === "PICTURE" ? 'im_' : 'oe_'
var srcset = []; : elem.tagName === "IMG" ? 'im_' : 'oe_';
for (var i = 0; i < dataSS.length; i++) {
elem = dataSS[i];
if (elem.tagName === 'SOURCE') {
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE' && elem.dataset.srcset) {
srcset.push({srcset: elem.dataset.srcset});
}
} else if (elem.dataset.srcset) {
srcset.push({srcset: elem.dataset.srcset});
}
}
if (srcset.length) {
this.postMessage({
'type': 'values',
'srcset': {'values': srcset, 'presplit': false},
'context': {
'docBaseURI': $wbwindow.document.baseURI
}
}, true);
}
};
AutoFetchWorker.prototype.extractImgPictureSourceSrcsets = function () {
var i;
var elem = null;
var srcset = [];
var ssElements = $wbwindow.document.querySelectorAll('img[srcset], source[srcset]');
for (i = 0; i < ssElements.length; i++) {
elem = ssElements[i];
if (elem.tagName === 'SOURCE') {
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE') {
srcset.push({srcset: this.extractSrcset(elem)});
}
} else {
srcset.push({tagSrc: elem.src, srcset: this.extractSrcset(elem)});
}
}
return srcset;
}; };
AutoFetchWorker.prototype.extractFromLocalDoc = function () { AutoFetchWorker.prototype.extractFromLocalDoc = function () {
// get the values to be preserved from the documents stylesheets // get the values to be preserved from the documents stylesheets
// and all elements with a srcset // and all img, video, audio elements with (data-)?srcset or data-src
var media = []; var afw = this;
var sheets = $wbwindow.document.styleSheets; Promise.resolve().then(function () {
var i = 0; var msg = { 'type': 'values', 'context': { 'docBaseURI': $wbwindow.document.baseURI } };
for (; i < sheets.length; ++i) { var media = [];
var rules = sheets[i].cssRules; var i = 0;
for (var j = 0; j < rules.length; ++j) { var sheets = $wbwindow.document.styleSheets;
var rule = rules[j]; for (; i < sheets.length; ++i) {
if (rule.type === CSSRule.MEDIA_RULE) { var rules = sheets[i].cssRules;
media.push(rule.cssText); for (var j = 0; j < rules.length; ++j) {
var rule = rules[j];
if (rule.type === CSSRule.MEDIA_RULE) {
media.push(rule.cssText);
}
} }
} }
} var elems = $wbwindow.document.querySelectorAll(afw.elemSelector);
var srcset = this.extractImgPictureSourceSrcsets(); var srcset = { 'values': [], 'presplit': false };
// send the extracted values to the worker deferred var src = { 'values': [] };
// to ensure the page viewer sees the images first var elem, srcv, mod;
this.postMessage({ for (i = 0; i < elems.length; ++i) {
'type': 'values', elem = elems[i];
'media': media, // we want the original src value in order to resolve URLs in the worker when needed
'srcset': {'values': srcset, 'presplit': false}, srcv = elem.src ? elem.src : null;
'context': { // a from value of 1 indicates images and a 2 indicates audio/video
'docBaseURI': $wbwindow.document.baseURI mod = afw.rwMod(elem);
if (elem.srcset) {
srcset.values.push({
'srcset': afw.getSrcset(elem),
'mod': mod,
'tagSrc': srcv
});
}
if (elem.dataset.srcset) {
srcset.values.push({
'srcset': elem.dataset.srcset,
'mod': mod,
'tagSrc': srcv
});
}
if (elem.dataset.src) {
src.values.push({'src': elem.dataset.src, 'mod': mod});
}
if (elem.tagName === "SOURCE" && srcv) {
src.values.push({'src': srcv, 'mod': mod});
}
}
if (media.length) {
msg.media = media;
}
if (srcset.values.length) {
msg.srcset = srcset;
}
if (src.values.length) {
msg.src = src;
}
if (msg.media || msg.srcset || msg.src) {
afw.postMessage(msg);
} }
}, true);
// deffer the checking of img/source data-srcset
// so that we do not clobber the UI thread
var self = this;
Promise.resolve().then(function () {
self.checkForPictureSourceDataSrcsets();
}); });
}; };
WBAutoFetchWorker = new AutoFetchWorker(wb_abs_prefix, wbinfo.mod); WBAutoFetchWorker = new AutoFetchWorker({
'prefix': wb_abs_prefix, 'mod': wbinfo.mod, 'rwRe': rwRe
});
wbSheetMediaQChecker = function checkStyle() { wbSheetMediaQChecker = function checkStyle() {
// used only for link[rel='stylesheet'] so we remove our listener // used only for link[rel='stylesheet'] so we remove our listener
@ -1680,7 +1697,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
} else if (name == "style") { } else if (name == "style") {
new_value = rewrite_style(value); new_value = rewrite_style(value);
} else if (name == "srcset") { } else if (name == "srcset") {
new_value = rewrite_srcset(value, isImageSrcset(elem)); new_value = rewrite_srcset(value, elem);
} else { } else {
// Only rewrite if absolute url // Only rewrite if absolute url
if (abs_url_only && !starts_with(value, VALID_PREFIXES)) { if (abs_url_only && !starts_with(value, VALID_PREFIXES)) {
@ -1688,8 +1705,8 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
var mod = rwModForElement(elem, name); var mod = rwModForElement(elem, name);
new_value = rewrite_url(value, false, mod, elem.ownerDocument); new_value = rewrite_url(value, false, mod, elem.ownerDocument);
if (wbUseAFWorker && isImageDataSrcset(elem)) { if (wbUseAFWorker && isSavedDataSrcSrcset(elem)) {
WBAutoFetchWorker.preserveDataSrcset(elem.dataset.srcset); WBAutoFetchWorker.preserveDataSrcset(elem);
} }
} }
@ -1704,7 +1721,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
function style_replacer(match, n1, n2, n3, offset, string) { function style_replacer(match, n1, n2, n3, offset, string) {
return n1 + rewrite_url(n2) + n3; return n1 + rewrite_url(n2) + n3;
} }
function rewrite_style(value) function rewrite_style(value)
{ {
if (!value) { if (!value) {
@ -1725,7 +1742,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
} }
//============================================ //============================================
function rewrite_srcset(value, isImage) function rewrite_srcset(value, elem)
{ {
if (!value) { if (!value) {
return ""; return "";
@ -1738,9 +1755,9 @@ var _WBWombat = function($wbwindow, wbinfo) {
values[i] = rewrite_url(values[i].trim()); values[i] = rewrite_url(values[i].trim());
} }
if (wbUseAFWorker && isImage) { if (wbUseAFWorker && isSavedSrcSrcset(elem)) {
// send post split values to preservation worker // send post split values to preservation worker
WBAutoFetchWorker.preserveSrcset(values); WBAutoFetchWorker.preserveSrcset(values, WBAutoFetchWorker.rwMod(elem));
} }
return values.join(", "); return values.join(", ");
} }
@ -1869,6 +1886,9 @@ var _WBWombat = function($wbwindow, wbinfo) {
changed = rewrite_attr(elem, 'src'); changed = rewrite_attr(elem, 'src');
changed = rewrite_attr(elem, 'srcset') || changed; changed = rewrite_attr(elem, 'srcset') || changed;
changed = rewrite_attr(elem, 'style') || changed; changed = rewrite_attr(elem, 'style') || changed;
if (wbUseAFWorker && elem.dataset.srcset) {
WBAutoFetchWorker.preserveDataSrcset(elem);
}
break; break;
case 'OBJECT': case 'OBJECT':
changed = rewrite_attr(elem, "data", true); changed = rewrite_attr(elem, "data", true);
@ -2097,7 +2117,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
if (mod == "cs_" && orig.indexOf("data:text/css") == 0) { if (mod == "cs_" && orig.indexOf("data:text/css") == 0) {
val = rewrite_inline_style(orig); val = rewrite_inline_style(orig);
} else if (attr == "srcset") { } else if (attr == "srcset") {
val = rewrite_srcset(orig, isImageSrcset(this)); val = rewrite_srcset(orig, this);
} else if (this.tagName === 'LINK' && attr === 'href') { } else if (this.tagName === 'LINK' && attr === 'href') {
var relV = this.rel; var relV = this.rel;
if (relV === 'import' || relV === 'preload') { if (relV === 'import' || relV === 'preload') {
@ -2213,7 +2233,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
override_style_attr(style_proto, "borderImageSource", "border-image-source"); override_style_attr(style_proto, "borderImageSource", "border-image-source");
override_style_setProp(style_proto); override_style_setProp(style_proto);
if ($wbwindow.CSSStyleSheet && $wbwindow.CSSStyleSheet.prototype) { if ($wbwindow.CSSStyleSheet && $wbwindow.CSSStyleSheet.prototype) {
// https://developer.mozilla.org/en-US/docs/Web/API/CSSStyleSheet/insertRule // https://developer.mozilla.org/en-US/docs/Web/API/CSSStyleSheet/insertRule
// ruleText is a string of raw css.... // ruleText is a string of raw css....
@ -2223,7 +2243,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
}; };
} }
} }
//============================================ //============================================
function override_style_setProp(style_proto) { function override_style_setProp(style_proto) {
var orig_setProp = style_proto.setProperty; var orig_setProp = style_proto.setProperty;
@ -2439,7 +2459,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
Object.defineProperty($wbwindow.FontFace.prototype, "constructor", {value: $wbwindow.FontFace}); Object.defineProperty($wbwindow.FontFace.prototype, "constructor", {value: $wbwindow.FontFace});
$wbwindow.FontFace.__wboverriden__ = true; $wbwindow.FontFace.__wboverriden__ = true;
} }
//============================================ //============================================
function overrideTextProtoGetSet(textProto, whichProp) { function overrideTextProtoGetSet(textProto, whichProp) {
var orig_getter = get_orig_getter(textProto, whichProp); var orig_getter = get_orig_getter(textProto, whichProp);
@ -2464,7 +2484,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
}; };
def_prop(textProto, whichProp, setter, getter); def_prop(textProto, whichProp, setter, getter);
} }
function overrideTextProtoFunction(textProto, whichFN) { function overrideTextProtoFunction(textProto, whichFN) {
var original = textProto[whichFN]; var original = textProto[whichFN];
textProto[whichFN] = function () { textProto[whichFN] = function () {
@ -2491,7 +2511,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
return original.apply(this, args); return original.apply(this, args);
}; };
} }
function initTextNodeOverrides($wbwindow) { function initTextNodeOverrides($wbwindow) {
if (!$wbwindow.Text || !$wbwindow.Text.prototype) return; if (!$wbwindow.Text || !$wbwindow.Text.prototype) return;
// https://dom.spec.whatwg.org/#characterdata and https://dom.spec.whatwg.org/#interface-text // https://dom.spec.whatwg.org/#characterdata and https://dom.spec.whatwg.org/#interface-text
@ -2507,7 +2527,7 @@ var _WBWombat = function($wbwindow, wbinfo) {
overrideTextProtoGetSet(textProto, 'data'); overrideTextProtoGetSet(textProto, 'data');
overrideTextProtoGetSet(textProto, 'wholeText'); overrideTextProtoGetSet(textProto, 'wholeText');
} }
//============================================ //============================================
function init_wombat_loc(win) { function init_wombat_loc(win) {
@ -3847,14 +3867,14 @@ var _WBWombat = function($wbwindow, wbinfo) {
initFontFaceOverride($wbwindow); initFontFaceOverride($wbwindow);
// Worker override (experimental) // Worker override (experimental)
initAutoFetchWorker(); initAutoFetchWorker(rx);
init_web_worker_override(); init_web_worker_override();
init_service_worker_override(); init_service_worker_override();
initSharedWorkerOverride(); initSharedWorkerOverride();
// text node overrides for js frameworks doing funky things with CSS // text node overrides for js frameworks doing funky things with CSS
initTextNodeOverrides($wbwindow); initTextNodeOverrides($wbwindow);
// innerHTML can be overriden on prototype! // innerHTML can be overriden on prototype!
override_html_assign($wbwindow.HTMLElement, "innerHTML", true); override_html_assign($wbwindow.HTMLElement, "innerHTML", true);
override_html_assign($wbwindow.HTMLElement, "outerHTML", true); override_html_assign($wbwindow.HTMLElement, "outerHTML", true);

205
pywb/static/wombatProxyMode.js
View File

@ -29,55 +29,56 @@ var _WBWombat = function ($wbwindow, wbinfo) {
wbinfo.wombat_opts = wbinfo.wombat_opts || {}; wbinfo.wombat_opts = wbinfo.wombat_opts || {};
var wbAutoFetchWorkerPrefix = (wb_info.auto_fetch_worker_prefix || wb_info.static_prefix) + 'autoFetchWorkerProxyMode.js'; var wbAutoFetchWorkerPrefix = (wb_info.auto_fetch_worker_prefix || wb_info.static_prefix) + 'autoFetchWorkerProxyMode.js';
var WBAutoFetchWorker; var WBAutoFetchWorker;
function init_seeded_random(seed) { function init_seeded_random(seed) {
// Adapted from: // Adapted from:
// http://indiegamr.com/generate-repeatable-random-numbers-in-js/ // http://indiegamr.com/generate-repeatable-random-numbers-in-js/
$wbwindow.Math.seed = parseInt(seed); $wbwindow.Math.seed = parseInt(seed);
function seeded_random() { function seeded_random() {
$wbwindow.Math.seed = ($wbwindow.Math.seed * 9301 + 49297) % 233280; $wbwindow.Math.seed = ($wbwindow.Math.seed * 9301 + 49297) % 233280;
var rnd = $wbwindow.Math.seed / 233280; var rnd = $wbwindow.Math.seed / 233280;
return rnd; return rnd;
} }
$wbwindow.Math.random = seeded_random; $wbwindow.Math.random = seeded_random;
} }
function init_crypto_random() { function init_crypto_random() {
if (!$wbwindow.crypto || !$wbwindow.Crypto) { if (!$wbwindow.crypto || !$wbwindow.Crypto) {
return; return;
} }
var orig_getrandom = $wbwindow.Crypto.prototype.getRandomValues; var orig_getrandom = $wbwindow.Crypto.prototype.getRandomValues;
var new_getrandom = function (array) { var new_getrandom = function (array) {
for (var i = 0; i < array.length; i++) { for (var i = 0; i < array.length; i++) {
array[i] = parseInt($wbwindow.Math.random() * 4294967296); array[i] = parseInt($wbwindow.Math.random() * 4294967296);
} }
return array; return array;
}; };
$wbwindow.Crypto.prototype.getRandomValues = new_getrandom; $wbwindow.Crypto.prototype.getRandomValues = new_getrandom;
$wbwindow.crypto.getRandomValues = new_getrandom; $wbwindow.crypto.getRandomValues = new_getrandom;
} }
//============================================ //============================================
function init_fixed_ratio() { function init_fixed_ratio() {
// otherwise, just set it // otherwise, just set it
$wbwindow.devicePixelRatio = 1; $wbwindow.devicePixelRatio = 1;
// prevent changing, if possible // prevent changing, if possible
if (Object.defineProperty) { if (Object.defineProperty) {
try { try {
// fixed pix ratio // fixed pix ratio
Object.defineProperty($wbwindow, "devicePixelRatio", {value: 1, writable: false}); Object.defineProperty($wbwindow, "devicePixelRatio", {value: 1, writable: false});
} catch (e) {} } catch (e) {
}
} }
} }
//======================================== //========================================
function init_date_override(timestamp) { function init_date_override(timestamp) {
timestamp = parseInt(timestamp) * 1000; timestamp = parseInt(timestamp) * 1000;
@ -86,19 +87,19 @@ var _WBWombat = function ($wbwindow, wbinfo) {
var timezone = 0; var timezone = 0;
var start_now = $wbwindow.Date.now(); var start_now = $wbwindow.Date.now();
var timediff = start_now - (timestamp - timezone); var timediff = start_now - (timestamp - timezone);
if ($wbwindow.__wb_Date_now) { if ($wbwindow.__wb_Date_now) {
return; return;
} }
var orig_date = $wbwindow.Date; var orig_date = $wbwindow.Date;
var orig_utc = $wbwindow.Date.UTC; var orig_utc = $wbwindow.Date.UTC;
var orig_parse = $wbwindow.Date.parse; var orig_parse = $wbwindow.Date.parse;
var orig_now = $wbwindow.Date.now; var orig_now = $wbwindow.Date.now;
$wbwindow.__wb_Date_now = orig_now; $wbwindow.__wb_Date_now = orig_now;
$wbwindow.Date = function (Date) { $wbwindow.Date = function (Date) {
return function (A, B, C, D, E, F, G) { return function (A, B, C, D, E, F, G) {
// Apply doesn't work for constructors and Date doesn't // Apply doesn't work for constructors and Date doesn't
@ -123,21 +124,21 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
} }
}($wbwindow.Date); }($wbwindow.Date);
$wbwindow.Date.prototype = orig_date.prototype; $wbwindow.Date.prototype = orig_date.prototype;
$wbwindow.Date.now = function () { $wbwindow.Date.now = function () {
return orig_now() - timediff; return orig_now() - timediff;
}; };
$wbwindow.Date.UTC = orig_utc; $wbwindow.Date.UTC = orig_utc;
$wbwindow.Date.parse = orig_parse; $wbwindow.Date.parse = orig_parse;
$wbwindow.Date.__WB_timediff = timediff; $wbwindow.Date.__WB_timediff = timediff;
Object.defineProperty($wbwindow.Date.prototype, "constructor", {value: $wbwindow.Date}); Object.defineProperty($wbwindow.Date.prototype, "constructor", {value: $wbwindow.Date});
} }
//============================================ //============================================
function init_disable_notifications() { function init_disable_notifications() {
if (window.Notification) { if (window.Notification) {
@ -145,36 +146,46 @@ var _WBWombat = function ($wbwindow, wbinfo) {
if (callback) { if (callback) {
callback("denied"); callback("denied");
} }
return Promise.resolve("denied"); return Promise.resolve("denied");
}; };
} }
if (window.geolocation) { if (window.geolocation) {
var disabled = function (success, error, options) { var disabled = function (success, error, options) {
if (error) { if (error) {
error({"code": 2, "message": "not available"}); error({"code": 2, "message": "not available"});
} }
}; };
window.geolocation.getCurrentPosition = disabled; window.geolocation.getCurrentPosition = disabled;
window.geolocation.watchPosition = disabled; window.geolocation.watchPosition = disabled;
} }
} }
function initAutoFetchWorker() { function initAutoFetchWorker() {
if (!$wbwindow.Worker) { if (!$wbwindow.Worker) {
return; return;
} }
var isTop = $wbwindow.self === $wbwindow.top; var isTop = $wbwindow.self === $wbwindow.top;
function AutoFetchWorkerProxyMode() { function AutoFetchWorkerProxyMode() {
if (!(this instanceof AutoFetchWorkerProxyMode)) { if (!(this instanceof AutoFetchWorkerProxyMode)) {
return new AutoFetchWorkerProxyMode(); return new AutoFetchWorkerProxyMode();
} }
this.checkIntervalTime = 15000; this.checkIntervalTime = 15000;
this.checkIntervalCB = this.checkIntervalCB.bind(this); this.checkIntervalCB = this.checkIntervalCB.bind(this);
this.elemSelector = ['img', 'source', 'video', 'audio'].map(function (which) {
if (which === 'source') {
return ['picture > ', 'video > ', 'audio >'].map(function (parent) {
return parent + which + '[srcset], ' + parent + which + '[data-srcset], ' + parent + which + '[data-src]'
}).join(', ');
} else {
return which + '[srcset], ' + which + '[data-srcset], ' + which + '[data-src]';
}
}).join(', ');
if (isTop) { if (isTop) {
// Cannot directly load our worker from the proxy origin into the current origin // Cannot directly load our worker from the proxy origin into the current origin
// however we fetch it from proxy origin and can blob it into the current origin :) // however we fetch it from proxy origin and can blob it into the current origin :)
@ -200,12 +211,13 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
$wbwindow.top.postMessage(msg, '*'); $wbwindow.top.postMessage(msg, '*');
}, },
"terminate": function () {} "terminate": function () {
}
}; };
this.startCheckingInterval(); this.startCheckingInterval();
} }
} }
AutoFetchWorkerProxyMode.prototype.startCheckingInterval = function () { AutoFetchWorkerProxyMode.prototype.startCheckingInterval = function () {
// if document ready state is complete do first extraction and start check polling // if document ready state is complete do first extraction and start check polling
// otherwise wait for document ready state to complete to extract and start check polling // otherwise wait for document ready state to complete to extract and start check polling
@ -223,16 +235,16 @@ var _WBWombat = function ($wbwindow, wbinfo) {
}, 1000); }, 1000);
} }
}; };
AutoFetchWorkerProxyMode.prototype.checkIntervalCB = function () { AutoFetchWorkerProxyMode.prototype.checkIntervalCB = function () {
this.extractFromLocalDoc(); this.extractFromLocalDoc();
}; };
AutoFetchWorkerProxyMode.prototype.terminate = function () { AutoFetchWorkerProxyMode.prototype.terminate = function () {
// terminate the worker, a no op when not replay top // terminate the worker, a no op when not replay top
this.worker.terminate(); this.worker.terminate();
}; };
AutoFetchWorkerProxyMode.prototype.postMessage = function (msg, deferred) { AutoFetchWorkerProxyMode.prototype.postMessage = function (msg, deferred) {
if (deferred) { if (deferred) {
var self = this; var self = this;
@ -242,7 +254,7 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
this.worker.postMessage(msg); this.worker.postMessage(msg);
}; };
AutoFetchWorkerProxyMode.prototype.extractMediaRules = function (rules, href) { AutoFetchWorkerProxyMode.prototype.extractMediaRules = function (rules, href) {
// We are in proxy mode and must include a URL to resolve relative URLs in media rules // We are in proxy mode and must include a URL to resolve relative URLs in media rules
if (!rules) return []; if (!rules) return [];
@ -257,7 +269,7 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
return text; return text;
}; };
AutoFetchWorkerProxyMode.prototype.corsCSSFetch = function (href) { AutoFetchWorkerProxyMode.prototype.corsCSSFetch = function (href) {
// because this JS in proxy mode operates as it would on the live web // because this JS in proxy mode operates as it would on the live web
// the rules of CORS apply and we cannot rely on URLs being rewritten correctly // the rules of CORS apply and we cannot rely on URLs being rewritten correctly
@ -274,70 +286,50 @@ var _WBWombat = function ($wbwindow, wbinfo) {
return []; return [];
}); });
}; };
AutoFetchWorkerProxyMode.prototype.shouldSkipSheet = function (sheet) { AutoFetchWorkerProxyMode.prototype.shouldSkipSheet = function (sheet) {
// we skip extracting rules from sheets if they are from our parsing style or come from pywb // we skip extracting rules from sheets if they are from our parsing style or come from pywb
if (sheet.id === '$wrStyleParser$') return true; if (sheet.id === '$wrStyleParser$') return true;
return !!(sheet.href && sheet.href.indexOf(wb_info.proxy_magic) !== -1); return !!(sheet.href && sheet.href.indexOf(wb_info.proxy_magic) !== -1);
}; };
AutoFetchWorkerProxyMode.prototype.extractImgPictureSourceSrcsets = function () { AutoFetchWorkerProxyMode.prototype.getImgAVElems = function () {
var i; var elem, srcv, mod;
var elem; var results = { 'srcset': [], 'src': []} ;
var srcset = [];
var baseURI = $wbwindow.document.baseURI; var baseURI = $wbwindow.document.baseURI;
var ssElements = $wbwindow.document.querySelectorAll('img[srcset], source[srcset]'); var elems = $wbwindow.document.querySelectorAll(this.elemSelector);
for (i = 0; i < ssElements.length; i++) { for (var i = 0; i < elems.length; i++) {
elem = ssElements[i]; elem = elems[i];
if (elem.tagName === 'SOURCE') { // we want the original src value in order to resolve URLs in the worker when needed
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE') { srcv = elem.src ? elem.src : null;
srcset.push({srcset: elem.srcset, resolve: baseURI}); // get the correct mod in order to inform the backing worker where the URL(s) are from
} mod = elem.tagName === "SOURCE" ?
} else { elem.parentElement.tagName === "PICTURE" ? 'im_' : 'oe_'
srcset.push({ : elem.tagName === "IMG" ? 'im_' : 'oe_';
srcset: elem.srcset, if (elem.srcset) {
resolve: elem.src != null && elem.src !== ' ' ? elem.src : baseURI results.srcset.push({ 'srcset': elem.srcset, 'resolve': srcv || baseURI, 'mod': mod });
}); }
if (elem.dataset.srcset) {
results.srcset.push({ 'srcset': elem.dataset.srcset, 'resolve': srcv || baseURI, 'mod': mod });
}
if (elem.dataset.src) {
results.src.push({'src': elem.dataset.src, 'resolve': srcv || baseURI, 'mod': mod});
}
if (elem.tagName === "SOURCE" && srcv) {
results.src.push({'src': srcv, 'resolve': baseURI, 'mod': mod});
} }
} }
return srcset; return results;
}; };
AutoFetchWorkerProxyMode.prototype.checkForPictureSourceDataSrcsets = function () {
var baseURI = $wbwindow.document.baseURI;
var dataSS = $wbwindow.document.querySelectorAll('img[data-srcset], source[data-srcset]');
var elem;
var srcset = [];
for (var i = 0; i < dataSS.length; i++) {
elem = dataSS[i];
if (elem.tagName === 'SOURCE') {
if (elem.parentElement && elem.parentElement.tagName === 'PICTURE' && elem.dataset.srcset) {
srcset.push({srcset: elem.dataset.srcset, resolve: baseURI});
}
} else if (elem.dataset.srcset) {
srcset.push({srcset: elem.dataset.srcset, resolve: elem.src != null && elem.src !== ' ' ? elem.src : baseURI});
}
}
if (srcset.length) {
this.postMessage({
'type': 'values',
'srcset': {'values': srcset, 'presplit': false},
'context': {
'docBaseURI': $wbwindow.document.baseURI
}
}, true);
}
};
AutoFetchWorkerProxyMode.prototype.extractFromLocalDoc = function () { AutoFetchWorkerProxyMode.prototype.extractFromLocalDoc = function () {
var i = 0;
var media = []; var media = [];
var deferredMediaURLS = []; var deferredMediaURLS = [];
var sheet; var sheet;
var resolve; var resolve;
// We must use the window reference passed to us to access this origins stylesheets // We must use the window reference passed to us to access this origins stylesheets
var styleSheets = $wbwindow.document.styleSheets; var styleSheets = $wbwindow.document.styleSheets;
for (; i < styleSheets.length; ++i) { for (var i = 0; i < styleSheets.length; i++) {
sheet = styleSheets[i]; sheet = styleSheets[i];
// if the sheet belongs to our parser node we must skip it // if the sheet belongs to our parser node we must skip it
if (!this.shouldSkipSheet(sheet)) { if (!this.shouldSkipSheet(sheet)) {
@ -360,13 +352,22 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
// We must use the window reference passed to us to access this origins elements with srcset attr // We must use the window reference passed to us to access this origins elements with srcset attr
// like cssRule handling we must include a URL to resolve relative URLs by // like cssRule handling we must include a URL to resolve relative URLs by
var srcset = this.extractImgPictureSourceSrcsets(); var results = this.getImgAVElems();
var msg = { 'type': 'values' };
// send what we have extracted, if anything, to the worker for processing // send what we have extracted, if anything, to the worker for processing
if (media.length > 0 || srcset.length > 0) { if (media.length > 0) {
this.postMessage({'type': 'values', 'media': media, 'srcset': srcset}, true); msg.media = media;
} }
if (results.srcset) {
msg.srcset = results.srcset;
}
if (results.src) {
msg.src = results.src;
}
if (msg.media || msg.srcset || msg.src) {
this.postMessage(msg);
}
if (deferredMediaURLS.length > 0) { if (deferredMediaURLS.length > 0) {
// wait for all our deferred fetching and extraction of cross origin // wait for all our deferred fetching and extraction of cross origin
// stylesheets to complete and then send those values, if any, to the worker // stylesheets to complete and then send those values, if any, to the worker
@ -381,16 +382,10 @@ var _WBWombat = function ($wbwindow, wbinfo) {
} }
}); });
} }
// deffer the checking of img/source data-srcset
// so that we do not clobber the UI thread
var self = this;
Promise.resolve().then(function () {
self.checkForPictureSourceDataSrcsets();
});
}; };
WBAutoFetchWorker = new AutoFetchWorkerProxyMode(); WBAutoFetchWorker = new AutoFetchWorkerProxyMode();
if (isTop) { if (isTop) {
$wbwindow.addEventListener("message", function (event) { $wbwindow.addEventListener("message", function (event) {
if (event.data && event.data.wb_type === 'aaworker') { if (event.data && event.data.wb_type === 'aaworker') {
@ -399,11 +394,11 @@ var _WBWombat = function ($wbwindow, wbinfo) {
}, false); }, false);
} }
} }
if (wbinfo.enable_auto_fetch && wbinfo.is_live) { if (wbinfo.enable_auto_fetch && wbinfo.is_live) {
initAutoFetchWorker(); initAutoFetchWorker();
} }
// proxy mode overrides // proxy mode overrides
// Random // Random
init_seeded_random(wbinfo.wombat_sec); init_seeded_random(wbinfo.wombat_sec);
@ -425,13 +420,13 @@ var _WBWombat = function ($wbwindow, wbinfo) {
window._WBWombat = _WBWombat; window._WBWombat = _WBWombat;
window._WBWombatInit = function(wbinfo) { window._WBWombatInit = function (wbinfo) {
if (!this._wb_wombat || !this._wb_wombat.actual) { if (!this._wb_wombat || !this._wb_wombat.actual) {
this._wb_wombat = new _WBWombat(this, wbinfo); this._wb_wombat = new _WBWombat(this, wbinfo);
this._wb_wombat.actual = true; this._wb_wombat.actual = true;
} else if (!this._wb_wombat) { } else if (!this._wb_wombat) {
console.warn("_wb_wombat missing!"); console.warn("_wb_wombat missing!");
} }
}; };