diff --git a/pywb/vueui/src/cdx-simulator/README.md b/pywb/vueui/src/cdx-simulator/README.md new file mode 100644 index 00000000..b9a4a7be --- /dev/null +++ b/pywb/vueui/src/cdx-simulator/README.md @@ -0,0 +1,41 @@ +# How to incorporate CDX Simulator + +Place following code snippets in **index.js** + +## Import `CDXQueryWorkerSimulator` Mock Class + +It is the mock class to the main javascript built-in `Worker` class: + + ``import { CDXQueryWorkerSimulator } from "./cdx-simulator/cdx-simulator";`` + +## Initialize `queryWorker` with Mock Class + +Update `const queryWorker = ...` initialization in `CDXLoader` class, `loadCDX()` method + +### by replacing it + +``` +const queryWorker = new CDXQueryWorkerSimulator(this.staticPrefix + "/queryWorker.js"); +``` + +### or by adding a conditional, so you can go back and forth between simulator and real CDX-data-loader: + +for example with a URL-hash flag conditional: + +``` +const queryWorker = new (window.location.hash.indexOf('cdx_simulate') >= 0 ? CDXQueryWorkerSimulator : Worker)(this.staticPrefix + "/queryWorker.js"); +``` + +NOTE: where if the url contains '#cdx_simulate' the mock simulator will be used; using a URL hash does not interfere with the main URL parsing of the PYWB app + +## Configure Simulation + +Add a **local** storage (from Chrome Dev Tools > Application > Local Storage) + +``` +{"count":5000, "yearStart":2020, "yearEnd":2022, "fetchTime":3000} +``` + +where `count` is the total records, yearStart and yearEnd are self-explanatory, and `fetchTime` is how long it should take + +![cdx loader config](pywb-vueui-cdx-simulator-config.jpg) \ No newline at end of file diff --git a/pywb/vueui/src/cdx-simulator/cdx-simulator.js b/pywb/vueui/src/cdx-simulator/cdx-simulator.js index 42d225d6..5527fed3 100644 --- a/pywb/vueui/src/cdx-simulator/cdx-simulator.js +++ b/pywb/vueui/src/cdx-simulator/cdx-simulator.js @@ -23,14 +23,7 @@ class CDXRecordFactory { // exaggerate max count per day, any day can hold up to 10th of the month's captures const maxPerDay = avgPerMonth/10; - let avgTimePerRecord = opts.fetchTime/total; // e.g. 1000 ms / 10,000 - let waitAtEveryNRecords = 1; - if (avgTimePerRecord < 1) { // < 1ms per records - waitAtEveryNRecords = Math.ceil(1/avgTimePerRecord); // invert - avgTimePerRecord = 1; - } else { // >= 1ms per record - avgTimePerRecord = Math.round(avgTimePerRecord); - } + let startTime = Math.floor(new Date().getTime()); let recordI = 0; for(let y=years[0]; y<=years[1]; y++) { @@ -42,23 +35,25 @@ class CDXRecordFactory { const times = {}; // make sure we save to hash to de-dupe for(let i=0; i { - setTimeout(() => { - resolve(true); - }, avgTimePerRecord); - }); - await p; - } - const newTime = Math.floor(Math.random()*3600*24); + const newTime = [Math.floor(Math.random()*24), Math.floor(Math.random()*60), Math.floor(Math.random()*60)].join(''); times[newTime] = 1; } Object.keys(times).sort().forEach(time => { - records.push({url, timestamp: dayTimestampPrefix+('000000'+time).substr(-6)}); + records.push({url, timestamp: dayTimestampPrefix+time}); }); } } } + let endTime = Math.floor(new Date().getTime()); + + if (opts.fetchTime && opts.fetchTime > endTime - startTime) { // wait till we reac fetchTime + const p = new Promise((resolve) => { + setTimeout(() => { + resolve(true); + }, (opts.fetchTime - (endTime - startTime))); + }); + await p; + } return records; } } diff --git a/pywb/vueui/src/cdx-simulator/pywb-vueui-cdx-simulator-config.jpg b/pywb/vueui/src/cdx-simulator/pywb-vueui-cdx-simulator-config.jpg new file mode 100644 index 00000000..e8a513b2 Binary files /dev/null and b/pywb/vueui/src/cdx-simulator/pywb-vueui-cdx-simulator-config.jpg differ diff --git a/pywb/vueui/src/index.js b/pywb/vueui/src/index.js index 13b11d23..2fac63c1 100644 --- a/pywb/vueui/src/index.js +++ b/pywb/vueui/src/index.js @@ -1,7 +1,6 @@ import appData from "./App.vue"; import { PywbData } from "./model.js"; -import { CDXQueryWorkerSimulator } from "./cdx-simulator/cdx-simulator"; import Vue from "vue/dist/vue.esm.browser"; @@ -101,8 +100,6 @@ class CDXLoader { } async loadCDX(queryURL) { - // Use this to test CDX Loader - // const queryWorker = new CDXQueryWorkerSimulator(this.staticPrefix + "/queryWorker.js"); const queryWorker = new Worker(this.staticPrefix + "/queryWorker.js"); const p = new Promise((resolve) => {