added CDX Loader simulator updates; removed from default index file

pywb/vueui/src/cdx-simulator/README.md

@@ -0,0 +1,41 @@
+# How to incorporate CDX Simulator
+
+Place following code snippets in **index.js**
+
+## Import  `CDXQueryWorkerSimulator` Mock Class
+
+It is the mock class to the main javascript built-in `Worker` class:
+
+   ``import { CDXQueryWorkerSimulator } from "./cdx-simulator/cdx-simulator";``
+
+## Initialize `queryWorker` with Mock Class
+
+Update `const queryWorker = ...` initialization in `CDXLoader` class, `loadCDX()` method
+   
+### by replacing it
+
+```
+const queryWorker = new CDXQueryWorkerSimulator(this.staticPrefix + "/queryWorker.js");
+```
+
+### or by adding a conditional, so you can go back and forth between simulator and real CDX-data-loader:
+
+for example with a URL-hash flag conditional:
+
+```
+const queryWorker = new (window.location.hash.indexOf('cdx_simulate') >= 0 ? CDXQueryWorkerSimulator : Worker)(this.staticPrefix + "/queryWorker.js");
+```
+
+NOTE: where if the url contains '#cdx_simulate' the mock simulator will be used; using a URL hash does not interfere with the main URL parsing of the PYWB app
+
+## Configure Simulation 
+
+Add a **local** storage (from Chrome Dev Tools > Application > Local Storage)
+
+```
+{"count":5000, "yearStart":2020, "yearEnd":2022, "fetchTime":3000}
+```
+
+where `count` is the total records, yearStart and yearEnd are self-explanatory, and `fetchTime` is how long it should take
+
+![cdx loader config](pywb-vueui-cdx-simulator-config.jpg)

pywb/vueui/src/cdx-simulator/cdx-simulator.js

@@ -23,14 +23,7 @@ class CDXRecordFactory {
     // exaggerate max count per day, any day can hold up to 10th of the month's captures
     const maxPerDay = avgPerMonth/10;
 
-    let avgTimePerRecord = opts.fetchTime/total; // e.g. 1000 ms / 10,000
+    let startTime = Math.floor(new Date().getTime());
-    let waitAtEveryNRecords = 1;
-    if (avgTimePerRecord < 1) { // < 1ms per records
-      waitAtEveryNRecords = Math.ceil(1/avgTimePerRecord); // invert
-      avgTimePerRecord = 1;
-    } else { // >= 1ms per record
-      avgTimePerRecord = Math.round(avgTimePerRecord);
-    }
     let recordI = 0;
 
     for(let y=years[0]; y<=years[1]; y++) {
@@ -42,23 +35,25 @@ class CDXRecordFactory {
 
           const times = {}; // make sure we save to hash to de-dupe
           for(let i=0; i<timesCount; i++) {
-            if (recordI++ % waitAtEveryNRecords === 0) { // wait
+            const newTime = [Math.floor(Math.random()*24), Math.floor(Math.random()*60), Math.floor(Math.random()*60)].join('');
-              const p = new Promise((resolve) => {
-                setTimeout(() => {
-                  resolve(true);
-                }, avgTimePerRecord);
-              });
-              await p;
-            }
-            const newTime = Math.floor(Math.random()*3600*24);
             times[newTime] = 1;
           }
           Object.keys(times).sort().forEach(time => {
-            records.push({url, timestamp: dayTimestampPrefix+('000000'+time).substr(-6)});
+            records.push({url, timestamp: dayTimestampPrefix+time});
           });
         }
       }
     }
+    let endTime = Math.floor(new Date().getTime());
+
+    if (opts.fetchTime && opts.fetchTime > endTime - startTime) { // wait till we reac fetchTime
+      const p = new Promise((resolve) => {
+        setTimeout(() => {
+          resolve(true);
+        }, (opts.fetchTime - (endTime - startTime)));
+      });
+      await p;
+    }
     return records;
   }
 }

pywb/vueui/src/index.js

@@ -1,7 +1,6 @@
 import appData from "./App.vue";
 
 import { PywbData } from "./model.js";
-import { CDXQueryWorkerSimulator } from "./cdx-simulator/cdx-simulator";
 
 import Vue from "vue/dist/vue.esm.browser";
 
@@ -101,8 +100,6 @@ class CDXLoader {
   }
 
   async loadCDX(queryURL) {
-    // Use this to test CDX Loader
-    // const queryWorker = new CDXQueryWorkerSimulator(this.staticPrefix + "/queryWorker.js");
     const queryWorker = new Worker(this.staticPrefix + "/queryWorker.js");
 
     const p = new Promise((resolve) => {