Use JSON values in query string for JSON request bodies (#893)

This commit also adds a more complicated JSON test case that is
also in warcio.js to ensure parity.

Treat numbers like JavaScript's Number.prototype.toString() by
dropping decimal from floats if they represent whole number.

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 3
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
       - name: checkout

.github/workflows/publish_pypi.yaml

@@ -0,0 +1,33 @@
+name: Publish to PYPI
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-release:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v1
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: python -m pip install --upgrade pip wheel twine
+
+      - name: Build Dist
+        run: python setup.py sdist bdist_wheel --universal
+
+      - name: Publish package to TestPyPI
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+

.github/workflows/release.yaml

@@ -38,5 +38,6 @@ jobs:
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
 

.gitignore

@@ -53,3 +53,7 @@ git_hash.py
 
 # Sphinx documentation
 docs/_build/*
+
+# virtualenvs
+env/
+venv/

CHANGES.rst

@@ -1,3 +1,107 @@
+pywb 2.7.3 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* issue_792 catch warcio exception by @oskarhek in https://github.com/webrecorder/pywb/pull/793
+* Add ui.logo_home_url as config.yaml option by @tw4l in https://github.com/webrecorder/pywb/pull/791
+* [#795] Show error when adding duplicate warc file by @kuechensofa in https://github.com/webrecorder/pywb/pull/797
+* Make search page more intuitive by @krakan in https://github.com/webrecorder/pywb/pull/794
+* Modify search template buttons by @tw4l in https://github.com/webrecorder/pywb/pull/801
+* [#804] Use default_locale when lang not set in the request by @krakan in https://github.com/webrecorder/pywb/pull/805
+* feat: regex substitution on surt rules match by @mijho in https://github.com/webrecorder/pywb/pull/780
+* Bump minimatch from 3.0.4 to 3.1.2 in /pywb/vueui by @dependabot in https://github.com/webrecorder/pywb/pull/777
+* Bump decode-uri-component from 0.2.0 to 0.2.2 in /pywb/vueui by @dependabot in https://github.com/webrecorder/pywb/pull/786
+* rules: add 'debugNoBatch' rewrite for fb and insta by @ikreymer in https://github.com/webrecorder/pywb/pull/806
+* Vue main order by @tw4l in https://github.com/webrecorder/pywb/pull/809
+* wombat: bump to 3.4.4 https://github.com/webrecorder/pywb/pull/808
+
+pywb 2.7.2 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* Fix regression introduced by improper wombat update in 2.7.1
+* Fix `redirect_to_exact: false` functionality: if not set, UI will stay on current timestamp, but will display info on actual capture.
+* Location bar nav now keeps current timestamp instead of defaulting to calendar view.
+* 'Live' mode fixes, no longer cache live cdx entry, don't add timestamp when navigating in live mode without timestamp
+* Calendar dropdown on replay now scrollable.
+* Timeline toggle on replay is 'sticky', will stay on if toggled on replay.
+* Capture text: use '|' as in 'Current Capture: [title] | [capture date]'
+* Document title: Add 'Archived Page: ' prefix to avoid confusion with live pages.
+
+pywb 2.7.1 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* Add locale-dependent handling of first day of week by @krakan in https://github.com/webrecorder/pywb/pull/781
+* Make filter expressions translatable by @krakan in https://github.com/webrecorder/pywb/pull/783
+* Add title to top frame in framed replay
+* Add missing tooltip translation strings
+* Fix calendar and timeline rendering for replay URLs without a timestamp
+* Update template documentation
+
+pywb 2.7.0 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* New banner and calendar implementation in Vue.js, which supports localization/internationalization and easier local theming by @vanecat @ikreymer @tw4l with helpful feedback from @ldko
+* New interactive timeline to assist in navigating between captures
+* Add basic development Docker Compose configuration file
+* Update documentation
+* Add contributing guide
+
+pywb 2.6.9 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* eval rewrite update + latest wombat by @ikreymer in https://github.com/webrecorder/pywb/pull/763
+* Rewrite: Support target rewriting, open new windows in top-frame instead by @tw4l in https://github.com/webrecorder/pywb/pull/767
+* Add arm64 platform support by @luandro in https://github.com/webrecorder/pywb/pull/775
+* Add uwsgi virtualenv information by @tw4l in https://github.com/webrecorder/pywb/pull/770
+* update to wombat 3.3.11 to support additional replay improvements
+* automated pypi publish on release https://github.com/webrecorder/pywb/pull/776
+
+pywb 2.6.8 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* Upgrade webassets to v2.0 by @m4rk3r in https://github.com/webrecorder/pywb/pull/730
+* Encoding image 'srcset'  value including the intrinsic width by @yasarkunduz in https://github.com/webrecorder/pywb/pull/712
+* Prevent jinja2 from escaping HTML markup in collection metadata by @tw4l in https://github.com/webrecorder/pywb/pull/747
+* Increase uwsgi_buffer_size for nginx config by @edsu in https://github.com/webrecorder/pywb/pull/716
+* Add missing translation for the filter-epression field placeholder by @krakan in https://github.com/webrecorder/pywb/pull/721
+* Activate field validation when expanding the advanced options by @krakan in https://github.com/webrecorder/pywb/pull/722
+* S3 loader to use boto3 built-in credential configuration by @sebastian-nagel in https://github.com/webrecorder/pywb/pull/723
+* describing installation using pip by @sepastian in https://github.com/webrecorder/pywb/pull/726
+* Add missing org/image to docker run commands by @heyvito in https://github.com/webrecorder/pywb/pull/733
+* Format error messages by @edsu in https://github.com/webrecorder/pywb/pull/737
+* Ensure CDX status is a string by @edsu in https://github.com/webrecorder/pywb/pull/739
+* Improve replay banner's accessibility by @lwrubel in https://github.com/webrecorder/pywb/pull/742
+* Revisit headers load fix by @ikreymer in https://github.com/webrecorder/pywb/pull/751
+* Enable translation for the remaining strings on the search results page by @krakan in https://github.com/webrecorder/pywb/pull/752
+* revisit of redirect optimization: by @ikreymer in https://github.com/webrecorder/pywb/pull/753
+* proxy: add COEP header for proxy mode to avoid errors by @ikreymer in https://github.com/webrecorder/pywb/pull/755
+* tests run improvements: update from python setup.py test -> tox  by @ikreymer in https://github.com/webrecorder/pywb/pull/754
+* rewrite: detect edge-case where html starts with bom followed  by @ikreymer in https://github.com/webrecorder/pywb/pull/758
+* tests options: add PYWB_NO_VERIFY_SSL env var for tests to avoid fail… by @ikreymer in https://github.com/webrecorder/pywb/pull/760
+* rewriting fix: twitter video in embedded tweets by @ikreymer in https://github.com/webrecorder/pywb/pull/761
+* Add ir_ modifier by @ikreymer in https://github.com/webrecorder/pywb/pull/759
+* Remove unused Appveyor badge
+
+pywb 2.6.7 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* dependency: bump gevent to latest (21.12.0)
+* rewrite: fix eval rewriting where '._eval' was accidentally being rewritten
+* post-to-get conversion: properly handle json with top-level lists, to match cdxj-indexer, print parse errors, fixes `#709 <https://github.com/webrecorder/pywb/pull/709>`_
+
+pywb 2.6.6 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* dependency: don't use obsolete werkzeug useragent package `#704 <https://github.com/webrecorder/pywb/pull/704>`_
+* fix user-agent detection: use ua-parser module, default to new js-proxy mode, unless older browser detected `#707 <https://github.com/webrecorder/pywb/pull/707>`_
+* fix tests: disable broken s3 tests for now
+* Dockerfile: use python 3.8 by default
+
+pywb 2.6.5 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* fix build: add 'markupsafe<2.1.0' to requirements
+
+
 pywb 2.6.4 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -1077,7 +1181,7 @@ pywb 0.9.6 changelist
 pywb 0.9.5 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 
-* s3 loading: support ``s3://`` scheme in block loader, allowing for loading index and archive files from s3. ``boto`` library must be installed seperately
+* s3 loading: support ``s3://`` scheme in block loader, allowing for loading index and archive files from s3. ``boto`` library must be installed separately
   via ``pip install boto``. Attempt default boto auth path, and if that fails, attempt anonymous s3 connection.
   
 * Wombat/Client-Side Rewrite Customizations: New ``rewrite_opts.client`` settings from ``config.yaml`` are passed directly to wombat as json. 
@@ -1173,7 +1277,7 @@ pywb 0.9.1 changelist
 
 * cdx server query: add support for ``url=*.host`` and ``url=host/*`` as shortcuts for ``matchType=domain`` and ``matchType=prefix``
 
-* zipnum cdx cluster: support loading index shared from prefix path instead of seperate location file.
+* zipnum cdx cluster: support loading index shared from prefix path instead of separate location file.
 
   The ``shard_index_loc`` config property may contain match and replace properties.
   Regex replacement is then used to obtain path prefix from the shard prefix path.
@@ -1539,7 +1643,7 @@ pywb 0.4.7 changelist
 
 * Rewrite: Parsing of html as raw bytes instead of decode/encode, detection still needed for non-ascii compatible encoding.
 
-* Indexing: Refactoring of cdx-indexer using a seperate 'archive record iterator' and pluggable cdx writer classes. Groundwork for creating custom indexers.
+* Indexing: Refactoring of cdx-indexer using a separate 'archive record iterator' and pluggable cdx writer classes. Groundwork for creating custom indexers.
 
 * Indexing: Support for 9 field cdx formats with -9 flag.
 

CONTRIBUTING.md

@@ -0,0 +1,68 @@
+# pywb contributing guide
+
+Thank you for your interest in contributing to pywb and open source web archiving tools!
+
+If you have a question not covered below or are interesting in collaborating, please feel free to reach out via any of our [contact points](https://webrecorder.net/contact).
+
+## How to contribute to pywb
+
+### I found a bug
+
+Please take a look at the [open issues](https://github.com/webrecorder/pywb/issues) to see if someone else has already described the same issue and if so, leave any comments or suggestions there.
+
+If no such issue already exists, feel free to [open a new issue](https://github.com/webrecorder/pywb/issues/new/choose) using the Bug Report template. If the bug is specifically related to replay of a particular site, instead use the Replay Issue template.
+
+When opening an issue or commenting on an open issue, please describe the problem you are having, any steps required to reproduce the bug (including the pywb version affected), and include any contextual information or screenshots that may be helpful.
+
+### I wrote a patch to fix a bug
+
+Please open a new pull request with a description of the changes and a link to the related issue (if no issue yet exists, please create one first).
+
+Create a new branch with a short descriptive name including the issue number, based on the latest `main` branch.
+
+All changes should be submitted with test coverage for the change as well as updates to the project documentation if appropriate.
+
+Avoid making unnecessary changes such as reformatting code or otherwise touching parts of the codebase that are not directly relevant to the issue at hand.
+
+We do our best to review pull requests in a timely manner but as we are a small team with many projects we cannot guarantee a response or merging timeline. Webrecorder reserves the right to reject pull requests that do not fit the direction of the project or ethics of the Webrecorder project.
+
+The Development section below has information on how to get started with working on pywb in a local development environment.
+
+### I want to propose a new feature
+
+Please take a look at the [open issues](https://github.com/webrecorder/pywb/issues) to see if someone else has already proposed a similar feature and if so, leave any comments or suggestions there.
+
+If no such issue already exists, feel free to [open a new issue](https://github.com/webrecorder/pywb/issues/new/choose) using the Feature Request template.
+
+## Development
+
+The [pywb documentation](https://pywb.readthedocs.io/en/latest/) contains information on pywb's architecture, configuration file, and how to get started with the software locally or in a Docker container.
+
+The project root directory contains a basic [Docker Compose](https://docs.docker.com/compose/) configuration file, which can be used to easily start a development environment. After installing Docker Desktop and Docker Compose (if not installed with Desktop), to run pywb in detached mode on `localhost:8080`, run:
+
+```bash
+docker compose up -d
+```
+
+(Note: this example assumes a newer version of Docker Desktop. For older versions that did not bundle Compose, you may need to replace `docker compose` with `docker-compose`)
+
+The first time you run this command, it make take some time to build.
+
+Changes to the [Vue](https://vuejs.org/) frontend components require rebuilding the Vue bundle (`pywb/static/vue/vueui.js`) to take effect. After making changes to one or more Vue components, you can rebuild the static bundle and view the changes in your development environment like so:
+
+```bash
+./build-vue-ui.sh
+docker compose up -d --build --force-recreate
+```
+
+Changes that modify pywb's Python dependencies or the operating system also require rebuilding the container:
+
+```bash
+docker compose up -d --build --force-recreate
+```
+
+To stop the container:
+
+```bash
+docker compose down
+```

Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON=python:3.7.2
+ARG PYTHON=python:3.8
 
 FROM $PYTHON
 

README.rst

@@ -1,13 +1,11 @@
-Webrecorder pywb 2.6
+Webrecorder pywb 2.8
 ====================
 
-.. image:: https://raw.githubusercontent.com/webrecorder/pywb/master/pywb/static/pywb-logo.png
+.. image:: https://raw.githubusercontent.com/webrecorder/pywb/main/pywb/static/pywb-logo.png
 
 .. image:: https://github.com/webrecorder/pywb/workflows/CI/badge.svg
       :target: https://github.com/webrecorder/pywb/actions
-.. image:: https://ci.appveyor.com/api/projects/status/qxnbunw65o929599/branch/master?svg=true
-      :target: https://ci.appveyor.com/project/webrecorder/pywb/branch/master
-.. image:: https://codecov.io/gh/webrecorder/pywb/branch/master/graph/badge.svg
+.. image:: https://codecov.io/gh/webrecorder/pywb/branch/main/graph/badge.svg
       :target: https://codecov.io/gh/webrecorder/pywb
 
 Web Archiving Tools for All
@@ -15,7 +13,7 @@ Web Archiving Tools for All
 
 `View the full pywb documentation <https://pywb.readthedocs.org>`_
 
-**pywb** is a Python (2 and 3) web archiving toolkit for replaying web archives large and small as accurately as possible.
+**pywb** is a Python 3 web archiving toolkit for replaying web archives large and small as accurately as possible.
 The toolkit now also includes new features for creating high-fidelity web archives.
 
 This toolset forms the foundation of Webrecorder project, but also provides a generic web archiving toolkit
@@ -51,23 +49,38 @@ The 2.x release included a major overhaul of pywb and introduces many new featur
 
 * New in 2.6: Support for localization and multi-language deployment.
 
+* New in 2.7: New banner/calendar UI written in `Vue <https://vuejs.org/>`_, with interactive timeline and easier theming of colors and logo via ``config.yaml``.
+
 
 Please see the `full documentation <https://pywb.readthedocs.org>`_ for more detailed info on all these features.
 
 
-Installation
-------------
+Installation for Deployment
+---------------------------
 
-To run and install locally you can:
+To install pywb for usage, you can use:
 
-* Install with ``python setup.py install``
+``pip install pywb``
 
-* Run tests with ``python setup.py test``
+Note: depending on your Python installation, you may have to use `pip3` instead of `pip`.
 
-* Run Wayback with ``wayback`` (see docs for info on how to setup collections)
 
-* Build docs locally with:  ``cd docs; make html``. (The docs will be built in ``./_build/html/index.html``)
+Installation from local copy
+----------------------------
 
+``git clone https://github.com/webrecorder/pywb``
+
+To install from a locally cloned copy, install with ``pip install -e .`` or ``python setup.py install``.
+
+To run tests, we recommend installing ``pip install tox tox-current-env`` and then running ``tox --current-env`` to test in your current Python environment.
+
+To Build docs locally, run:  ``cd docs; make html``. (The docs will be built in ``./_build/html/index.html``)
+
+
+Running
+-------
+
+After installation, you can run ``pywb`` or ``wayback``.
 
 Consult the local or `online docs <https://pywb.readthedocs.org>`_ for latest usage and configuration details.
 
@@ -91,10 +104,4 @@ The pywb documentation is extensive. Some links to a few key guides:
 Contributions & Bug Reports
 ---------------------------
 
-Users are encouraged to fork and contribute to this project to keep improving web archiving tools.
-
-If you are interested in contributing, especially to any of these areas, please let us know!
-
-Otherwise, please take a look at `list of current issues <https://github.com/webrecorder/pywb/issues>`_ and feel free to open new ones about any aspect of pywb, including the new documentation.
-
-
+Users are encouraged to fork and contribute to this project to keep improving web archiving tools. Please consult the `contributing guide <CONTRIBUTING.md>`_ for information on how to contribute to pywb.

build-vue-ui.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+CURR_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+cd $CURR_DIR/pywb/vueui/
+yarn install
+yarn run build

config.yaml

@@ -1,6 +1,16 @@
 # pywb config file
 # ========================================
 #
+debug: true
+
+# Uncomment to set banner colors and logo
+# ui:
+  # logo: path/relative/from/static/logo.png
+  # logo_home_url: https://example.com
+  # navbar_background_hex: 0c49b0
+  # navbar_color_hex: fff
+  # navbar_light_buttons: true
+  # disable_printing: true
 
 collections:
     all: $all
@@ -11,18 +21,18 @@ collections:
 # Settings for each collection
 use_js_obj_proxy: true
 
-# Memento support, enable
+# Eanable Memento support
 enable_memento: true
 
 # Replay content in an iframe
 framed_replay: true
 
+redirect_to_exact: true
 
-# uncomment and change to set default locale
+# Uncomment and change to set default locale
 # default_locale: en
 
-# uncomment to set available locales
-locales:
-  - en
-  - fr
-
+# Uncomment to set available locales
+# locales:
+#   - en
+#   - ru

docker-compose.yaml

@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  pywb:
+    build: .
+    ports:
+        - 8080:8080
+    volumes:
+        - ./config.yaml:/webarchive/config.yaml
+        - ./sample_archive/:/webarchive/sample_archive/

docs/code/pywb.apps.rst

@@ -1,78 +1,77 @@
-pywb\.apps package
-==================
+pywb.apps package
+=================
 
 Submodules
 ----------
 
-pywb\.apps\.cli module
-----------------------
+pywb.apps.cli module
+--------------------
 
 .. automodule:: pywb.apps.cli
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.frontendapp module
-------------------------------
+pywb.apps.frontendapp module
+----------------------------
 
 .. automodule:: pywb.apps.frontendapp
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.live module
------------------------
+pywb.apps.live module
+---------------------
 
 .. automodule:: pywb.apps.live
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.rewriterapp module
-------------------------------
+pywb.apps.rewriterapp module
+----------------------------
 
 .. automodule:: pywb.apps.rewriterapp
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.static\_handler module
-----------------------------------
-
-.. automodule:: pywb.apps.static_handler
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.apps\.warcserverapp module
+pywb.apps.static\_handler module
 --------------------------------
 
-.. automodule:: pywb.apps.warcserverapp
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.apps.static_handler
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.wayback module
---------------------------
+pywb.apps.warcserverapp module
+------------------------------
+
+.. automodule:: pywb.apps.warcserverapp
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.apps.wayback module
+------------------------
 
 .. automodule:: pywb.apps.wayback
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.apps\.wbrequestresponse module
-------------------------------------
+pywb.apps.wbrequestresponse module
+----------------------------------
 
 .. automodule:: pywb.apps.wbrequestresponse
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.apps
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.indexer.rst

@@ -1,30 +1,29 @@
-pywb\.indexer package
-=====================
+pywb.indexer package
+====================
 
 Submodules
 ----------
 
-pywb\.indexer\.archiveindexer module
-------------------------------------
+pywb.indexer.archiveindexer module
+----------------------------------
 
 .. automodule:: pywb.indexer.archiveindexer
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.indexer\.cdxindexer module
---------------------------------
+pywb.indexer.cdxindexer module
+------------------------------
 
 .. automodule:: pywb.indexer.cdxindexer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.indexer
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.manager.rst

@@ -1,46 +1,53 @@
-pywb\.manager package
-=====================
+pywb.manager package
+====================
 
 Submodules
 ----------
 
-pywb\.manager\.aclmanager module
---------------------------------
+pywb.manager.aclmanager module
+------------------------------
 
 .. automodule:: pywb.manager.aclmanager
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.manager\.autoindex module
--------------------------------
+pywb.manager.autoindex module
+-----------------------------
 
 .. automodule:: pywb.manager.autoindex
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.manager\.manager module
------------------------------
+pywb.manager.locmanager module
+------------------------------
+
+.. automodule:: pywb.manager.locmanager
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.manager.manager module
+---------------------------
 
 .. automodule:: pywb.manager.manager
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.manager\.migrate module
------------------------------
+pywb.manager.migrate module
+---------------------------
 
 .. automodule:: pywb.manager.migrate
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.manager
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.recorder.rst

@@ -1,46 +1,45 @@
-pywb\.recorder package
-======================
+pywb.recorder package
+=====================
 
 Submodules
 ----------
 
-pywb\.recorder\.filters module
-------------------------------
+pywb.recorder.filters module
+----------------------------
 
 .. automodule:: pywb.recorder.filters
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.recorder\.multifilewarcwriter module
-------------------------------------------
+pywb.recorder.multifilewarcwriter module
+----------------------------------------
 
 .. automodule:: pywb.recorder.multifilewarcwriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.recorder\.recorderapp module
-----------------------------------
+pywb.recorder.recorderapp module
+--------------------------------
 
 .. automodule:: pywb.recorder.recorderapp
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.recorder\.redisindexer module
------------------------------------
+pywb.recorder.redisindexer module
+---------------------------------
 
 .. automodule:: pywb.recorder.redisindexer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.recorder
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.rewrite.rst

@@ -1,150 +1,149 @@
-pywb\.rewrite package
-=====================
+pywb.rewrite package
+====================
 
 Submodules
 ----------
 
-pywb\.rewrite\.content\_rewriter module
----------------------------------------
+pywb.rewrite.content\_rewriter module
+-------------------------------------
 
 .. automodule:: pywb.rewrite.content_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.rewrite\.cookie\_rewriter module
---------------------------------------
-
-.. automodule:: pywb.rewrite.cookie_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.cookies module
------------------------------
-
-.. automodule:: pywb.rewrite.cookies
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.default\_rewriter module
----------------------------------------
-
-.. automodule:: pywb.rewrite.default_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.header\_rewriter module
---------------------------------------
-
-.. automodule:: pywb.rewrite.header_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.html\_insert\_rewriter module
---------------------------------------------
-
-.. automodule:: pywb.rewrite.html_insert_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.html\_rewriter module
+pywb.rewrite.cookie\_rewriter module
 ------------------------------------
 
-.. automodule:: pywb.rewrite.html_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.rewrite.cookie_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.rewrite\.jsonp\_rewriter module
--------------------------------------
-
-.. automodule:: pywb.rewrite.jsonp_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.regex\_rewriters module
---------------------------------------
-
-.. automodule:: pywb.rewrite.regex_rewriters
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.rewrite\_amf module
-----------------------------------
-
-.. automodule:: pywb.rewrite.rewrite_amf
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.rewrite\_dash module
------------------------------------
-
-.. automodule:: pywb.rewrite.rewrite_dash
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.rewrite\_hls module
-----------------------------------
-
-.. automodule:: pywb.rewrite.rewrite_hls
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.rewrite\_js\_workers module
-------------------------------------------
-
-.. automodule:: pywb.rewrite.rewrite_js_workers
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.rewriteinputreq module
--------------------------------------
-
-.. automodule:: pywb.rewrite.rewriteinputreq
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.templateview module
-----------------------------------
-
-.. automodule:: pywb.rewrite.templateview
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.url\_rewriter module
------------------------------------
-
-.. automodule:: pywb.rewrite.url_rewriter
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.rewrite\.wburl module
+pywb.rewrite.cookies module
 ---------------------------
 
-.. automodule:: pywb.rewrite.wburl
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.rewrite.cookies
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
+pywb.rewrite.default\_rewriter module
+-------------------------------------
+
+.. automodule:: pywb.rewrite.default_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.header\_rewriter module
+------------------------------------
+
+.. automodule:: pywb.rewrite.header_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.html\_insert\_rewriter module
+------------------------------------------
+
+.. automodule:: pywb.rewrite.html_insert_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.html\_rewriter module
+----------------------------------
+
+.. automodule:: pywb.rewrite.html_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.jsonp\_rewriter module
+-----------------------------------
+
+.. automodule:: pywb.rewrite.jsonp_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.regex\_rewriters module
+------------------------------------
+
+.. automodule:: pywb.rewrite.regex_rewriters
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.rewrite\_amf module
+--------------------------------
+
+.. automodule:: pywb.rewrite.rewrite_amf
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.rewrite\_dash module
+---------------------------------
+
+.. automodule:: pywb.rewrite.rewrite_dash
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.rewrite\_hls module
+--------------------------------
+
+.. automodule:: pywb.rewrite.rewrite_hls
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.rewrite\_js\_workers module
+----------------------------------------
+
+.. automodule:: pywb.rewrite.rewrite_js_workers
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.rewriteinputreq module
+-----------------------------------
+
+.. automodule:: pywb.rewrite.rewriteinputreq
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.templateview module
+--------------------------------
+
+.. automodule:: pywb.rewrite.templateview
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.url\_rewriter module
+---------------------------------
+
+.. automodule:: pywb.rewrite.url_rewriter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.rewrite.wburl module
+-------------------------
+
+.. automodule:: pywb.rewrite.wburl
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.rewrite
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.rst

@@ -5,31 +5,31 @@ Subpackages
 -----------
 
 .. toctree::
+   :maxdepth: 4
 
-    pywb.apps
-    pywb.indexer
-    pywb.manager
-    pywb.recorder
-    pywb.rewrite
-    pywb.utils
-    pywb.warcserver
+   pywb.apps
+   pywb.indexer
+   pywb.manager
+   pywb.recorder
+   pywb.rewrite
+   pywb.utils
+   pywb.warcserver
 
 Submodules
 ----------
 
-pywb\.version module
---------------------
+pywb.version module
+-------------------
 
 .. automodule:: pywb.version
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.utils.rst

@@ -1,86 +1,85 @@
-pywb\.utils package
-===================
+pywb.utils package
+==================
 
 Submodules
 ----------
 
-pywb\.utils\.binsearch module
------------------------------
+pywb.utils.binsearch module
+---------------------------
 
 .. automodule:: pywb.utils.binsearch
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.canonicalize module
---------------------------------
+pywb.utils.canonicalize module
+------------------------------
 
 .. automodule:: pywb.utils.canonicalize
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.format module
---------------------------
+pywb.utils.format module
+------------------------
 
 .. automodule:: pywb.utils.format
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.geventserver module
---------------------------------
+pywb.utils.geventserver module
+------------------------------
 
 .. automodule:: pywb.utils.geventserver
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.io module
-----------------------
+pywb.utils.io module
+--------------------
 
 .. automodule:: pywb.utils.io
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.loaders module
----------------------------
-
-.. automodule:: pywb.utils.loaders
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.utils\.memento module
----------------------------
-
-.. automodule:: pywb.utils.memento
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.utils\.merge module
+pywb.utils.loaders module
 -------------------------
 
-.. automodule:: pywb.utils.merge
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.utils.loaders
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.utils\.wbexception module
--------------------------------
+pywb.utils.memento module
+-------------------------
+
+.. automodule:: pywb.utils.memento
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.utils.merge module
+-----------------------
+
+.. automodule:: pywb.utils.merge
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.utils.wbexception module
+-----------------------------
 
 .. automodule:: pywb.utils.wbexception
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.utils
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.warcserver.index.rst

@@ -1,70 +1,69 @@
-pywb\.warcserver\.index package
-===============================
+pywb.warcserver.index package
+=============================
 
 Submodules
 ----------
 
-pywb\.warcserver\.index\.aggregator module
-------------------------------------------
+pywb.warcserver.index.aggregator module
+---------------------------------------
 
 .. automodule:: pywb.warcserver.index.aggregator
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.cdxobject module
------------------------------------------
+pywb.warcserver.index.cdxobject module
+--------------------------------------
 
 .. automodule:: pywb.warcserver.index.cdxobject
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.cdxops module
---------------------------------------
+pywb.warcserver.index.cdxops module
+-----------------------------------
 
 .. automodule:: pywb.warcserver.index.cdxops
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.fuzzymatcher module
---------------------------------------------
+pywb.warcserver.index.fuzzymatcher module
+-----------------------------------------
 
 .. automodule:: pywb.warcserver.index.fuzzymatcher
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.indexsource module
--------------------------------------------
+pywb.warcserver.index.indexsource module
+----------------------------------------
 
 .. automodule:: pywb.warcserver.index.indexsource
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.query module
--------------------------------------
+pywb.warcserver.index.query module
+----------------------------------
 
 .. automodule:: pywb.warcserver.index.query
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.index\.zipnum module
---------------------------------------
+pywb.warcserver.index.zipnum module
+-----------------------------------
 
 .. automodule:: pywb.warcserver.index.zipnum
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.warcserver.index
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.warcserver.resource.rst

@@ -1,46 +1,45 @@
-pywb\.warcserver\.resource package
-==================================
+pywb.warcserver.resource package
+================================
 
 Submodules
 ----------
 
-pywb\.warcserver\.resource\.blockrecordloader module
-----------------------------------------------------
-
-.. automodule:: pywb.warcserver.resource.blockrecordloader
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.resource\.pathresolvers module
-------------------------------------------------
-
-.. automodule:: pywb.warcserver.resource.pathresolvers
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.resource\.resolvingloader module
---------------------------------------------------
-
-.. automodule:: pywb.warcserver.resource.resolvingloader
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.resource\.responseloader module
+pywb.warcserver.resource.blockrecordloader module
 -------------------------------------------------
 
-.. automodule:: pywb.warcserver.resource.responseloader
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.warcserver.resource.blockrecordloader
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
+pywb.warcserver.resource.pathresolvers module
+---------------------------------------------
+
+.. automodule:: pywb.warcserver.resource.pathresolvers
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.warcserver.resource.resolvingloader module
+-----------------------------------------------
+
+.. automodule:: pywb.warcserver.resource.resolvingloader
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.warcserver.resource.responseloader module
+----------------------------------------------
+
+.. automodule:: pywb.warcserver.resource.responseloader
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.warcserver.resource
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/code/pywb.warcserver.rst

@@ -1,86 +1,86 @@
-pywb\.warcserver package
-========================
+pywb.warcserver package
+=======================
 
 Subpackages
 -----------
 
 .. toctree::
+   :maxdepth: 4
 
-    pywb.warcserver.index
-    pywb.warcserver.resource
+   pywb.warcserver.index
+   pywb.warcserver.resource
 
 Submodules
 ----------
 
-pywb\.warcserver\.access\_checker module
-----------------------------------------
+pywb.warcserver.access\_checker module
+--------------------------------------
 
 .. automodule:: pywb.warcserver.access_checker
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.amf module
-----------------------------
+pywb.warcserver.amf module
+--------------------------
 
 .. automodule:: pywb.warcserver.amf
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.basewarcserver module
----------------------------------------
-
-.. automodule:: pywb.warcserver.basewarcserver
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.handlers module
----------------------------------
-
-.. automodule:: pywb.warcserver.handlers
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.http module
------------------------------
-
-.. automodule:: pywb.warcserver.http
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-pywb\.warcserver\.inputrequest module
+pywb.warcserver.basewarcserver module
 -------------------------------------
 
-.. automodule:: pywb.warcserver.inputrequest
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.warcserver.basewarcserver
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.upstreamindexsource module
---------------------------------------------
+pywb.warcserver.handlers module
+-------------------------------
 
-.. automodule:: pywb.warcserver.upstreamindexsource
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.warcserver.handlers
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-pywb\.warcserver\.warcserver module
+pywb.warcserver.http module
+---------------------------
+
+.. automodule:: pywb.warcserver.http
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.warcserver.inputrequest module
 -----------------------------------
 
-.. automodule:: pywb.warcserver.warcserver
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: pywb.warcserver.inputrequest
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
+pywb.warcserver.upstreamindexsource module
+------------------------------------------
+
+.. automodule:: pywb.warcserver.upstreamindexsource
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+pywb.warcserver.warcserver module
+---------------------------------
+
+.. automodule:: pywb.warcserver.warcserver
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 Module contents
 ---------------
 
 .. automodule:: pywb.warcserver
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/conf.py

@@ -53,7 +53,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'pywb'
-copyright = '2014-2020, Webrecorder Software, Rhizome, and Contributors'
+copyright = '2014-2021, Webrecorder Software, Rhizome, and Contributors'
 author = 'Ilya Kreymer'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -61,9 +61,9 @@ author = 'Ilya Kreymer'
 # built documents.
 #
 # The short X.Y version.
-version = '2.0'
+version = '2.7'
 # The full version, including alpha/beta/rc tags.
-release = '2.0'
+release = '2.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/manual/access-control.rst

@@ -105,6 +105,12 @@ Given these rules, a user would:
 * but would receive an 'access blocked' error message when viewing ``http://httpbin.org/`` (block)
 * would receive a 404 not found error when viewing ``http://httpbin.org/anything`` (exclude)
 
+To match any possible URL in an .aclj file, set ``*,`` as the leading SURT, for example::
+
+  *, - {"access": "allow"}
+
+Lines starting with ``*,`` should generally be at the end of the file, respecting the reverse alphabetical order.
+
 
 Access Types: allow, block, exclude, allow_ignore_embargo
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -149,6 +155,10 @@ To make this work, pywb must be running behind an Apache or Nginx system that is
 
 For example, this header may be set based on IP range, or based on password authentication.
 
+To allow a user access to all URLs, overriding more specific rules and the ``default_access`` configuration setting, use the ``*,`` SURT::
+
+  *, - {"access": "allow", "user": "staff"}
+
 Further examples of how to set this header will be provided in the deployments section.
 
 **Note: Do not use the user-based rules without configuring proper authentication on an Apache or Nginx frontend to set or remove this header, otherwise the 'X-Pywb-ACL-User' can easily be faked.**

docs/manual/apps.rst

@@ -46,6 +46,7 @@ It can be used to:
 
 * Create a new collection --  ``wb-manager init <coll>``
 * Add WARCs to collection -- ``wb-manager add <coll> <warc>``
+* Unpack WACZs to add their WARCs and indices to collection -- ``wb-manager add --unpack-wacz <coll> <wacz>``
 * Add override templates
 * Add and remove metadata to a collections ``metadata.yaml``
 * List all collections

docs/manual/configuring.rst

@@ -16,8 +16,19 @@ With **framed replay**, the archived content is loaded into an iframe, and a top
 In this mode, the top frame url is for example, ``http://my-archive.example.com/<coll name>/http://example.com/`` while
 the actual content is served at ``http://my-archive.example.com/<coll name>/mp_/http://example.com/``
 
+With **frameless replay**, the archived content is loaded directly. As of pywb 2.7, frameless replay is bannerless
+unless a custom banner is added via the ``custom_banner.html`` template.
+
+
+.. warning::
+    pywb 2.7 introduces a breaking change around frameless replay and banners.
+    Any custom banner intended to be used with frameless replay in pywb 2.7 and
+    higher must be specified in the ``custom_banner.html`` template. This may
+    require moving custom content from ``banner.html`` to the new
+    ``custom_banner.html``.
+
+    The default banner will no longer be served in frameless replay.
 
-With **frameless replay**, the archived content is loaded directly, and a banner UI is injected into the page.
 
 In this mode, the content is served directly at ``http://my-archive.example.com/<coll name>/http://example.com/``
 

docs/manual/localization.rst

@@ -1,4 +1,4 @@
-.. _localizaation:
+.. _localization:
 
 Localization / Multi-lingual Support
 ------------------------------------

docs/manual/template-guide.rst

@@ -0,0 +1,367 @@
+.. _template-guide:
+
+Template Guide
+==============
+
+Introduction
+------------
+
+This guide provides a reference of all of the templates available in pywb and how they could be modified.
+
+These templates are found in the ``pywb/templates`` directory and can be overridden as needed, one HTML page at a time.
+
+Template variables are listed as ``{{ variable }}`` to indicate the syntax used for rendering the value of the variable in Jinja2.
+
+Copying a Template For Modification
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To modify a template, it is often useful to start with the default template. To do so, simply copy a default template
+to a local ``templates`` directory.
+
+For convenience, you can also run: ``wb-manager template --add <template-name>`` to add the template automatically.
+
+For a list of available templates that can be overridden in this way, run ``wb-manager template --list``.
+
+
+Per-Collection Templates
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Certain templates can be customized per-collection, instead of for all of pywb.
+
+To override a template for a specific collection only, run ``wb-manager template --add <template-name> <coll-name>``
+
+For example:
+
+
+.. code:: console
+    
+    wb-manager init my-coll
+    wb-manager template --add search_html my-coll
+
+This will create the file ``collections/my-coll/templates/search.html``, a copy of the default search.html, but configured to be used only
+for the collection ``my-coll``.
+
+
+
+Base Templates (and supporting templates)
+-----------------------------------------
+
+File: ``base.html``
+
+This template includes the HTML added to all pages other than framed replay. Shared JS and CSS includes meant for pages other than framed replay can be added here.
+
+To customize the default pywb UI across multiple pages, the following additional templates
+can also be overriden:
+
+* ``head.html`` -- Template containing content to be added to the ``<head>`` of the ``base`` template
+
+* ``header.html`` -- Template to be added as the first content of the ``<body>`` tag of the ``base`` template
+
+* ``footer.html`` -- Template for adding content as the "footer" of the ``<body>`` tag of the ``base`` template
+
+
+Note: The default pywb ``head.html`` and ``footer.html`` are currently blank. They can be populated to customize the rendering, add analytics, etc... as needed. Content such as styles or JS code (for example for analytics) must be added to the ``frame_insert.html`` template as well (details on that template below) to also be included in framed replay.
+
+
+The ``base.html`` template also provides five blocks that can be supplied by templates that extend it.
+
+* ``title`` -- Block for supplying the title for the page
+
+* ``head`` -- Block for adding content to the ``<head>``, includes ``head.html`` template
+
+* ``header`` -- Block for adding content to the ``<body>`` before the ``body`` block, includes the ``header.html`` template
+
+* ``body`` -- Block for adding the primary content to template
+
+* ``footer`` -- Block for adding content to the ``<body>`` after the ``body`` block, includes the ``footer.html`` template
+
+
+Home, Collection and Search Templates
+-------------------------------------
+
+
+Home Page Template
+^^^^^^^^^^^^^^^^^^
+
+File: ``index.html``
+
+This template renders the home page for pywb, and by default renders a list of available collections.
+
+
+Template variables:
+
+* ``{{ routes }}`` - a list of available collection routes.
+
+* ``{{ all_metadata }}`` - a dictionary of all metadata for all collections, keyed by collection id. See :ref:`custom-metadata` for more info on the custom metadata.
+
+
+Additionally, the :ref:`shared-template-vars` are also available to the home page template, as well as all other templates.
+
+
+Collection Page Template
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+File: ``search.html``
+
+The 'collection page' template is the page rendered when no URL is specified, e.g. ``http://localhost:8080/my-collection/``.
+
+The default template renders a search page that can be used to start searching for URLs.
+
+Template variables:
+
+* ``{{ coll }}`` - the collection name identifier.
+
+* ``{{ metadata }}`` - an optional dictionary of metadata. See :ref:`custom-metadata` for more info.
+
+* ``{{ ui }}`` - an optional ``ui`` dictionary from ``config.yaml``, if any
+
+
+.. _custom-metadata:
+
+Custom Metadata
+"""""""""""""""
+
+If custom collection metadata is provided, this page will automatically show this metadata as well.
+
+It is possible to also add custom metadata per-collection that will be available to the collection.
+
+For dynamic collections, any fields placed in ``<coll_name>/metadata.yaml`` files can be accessed
+
+via the ``{{ metadata }}`` variable.
+
+For example, if the metadata file contains:
+
+.. code:: yaml
+
+    somedata: value
+
+Accessing ``{{ metadata.somedata }}`` will resolve to ``value``.
+
+The metadata can also be added via commandline: ``wb-manager metadata myCollection --set somedata=value``.
+
+
+URL Query/Calendar Page Template
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+File: ``query.html``
+
+This template is rendered for any URL search response pages, either a single URL or more complex queries.
+
+For example, the page ``http://localhost:8080/my-collection/*/https://example.com/`` will be rendered using this template, with functionality provided by a Vue application.
+
+Template variables:
+
+* ``{{ url }}`` - the URL being queried, e.g. ``https://example.com/``
+
+* ``{{ prefix }}`` - the collection prefix that will be used for replay, e.g. ``http://localhost:8080/my-collection/``
+
+* ``{{ ui }}`` - an optional ``ui`` dictionary from ``config.yaml``, if any
+
+* ``{{ static_prefix }}`` - the prefix from which static files will be accessed from, e.g. ``http://localhost:8080/static/``.
+
+
+Replay and Banner Templates
+---------------------------
+
+The following templates are used to configure the replay view itself.
+
+
+Banner Template
+^^^^^^^^^^^^^^^
+
+File: ``banner.html``
+
+This template is used to render the banner for framed replay. It is rendered only rendered in the top/outer frame.
+
+Template variables:
+
+* ``{{ url }}`` - the URL being replayed.
+
+* ``{{ timestamp }}`` - the timestamp being replayed, e.g. ``20211226`` in ``http://localhost:8080/pywb/20211226/mp_/https://example.com/``
+
+* ``{{ is_framed }}`` - true/false if currently in framed mode.
+
+* ``{{ wb_prefix }}`` - the collection prefix, e.g. ``http://localhost:8080/pywb/``
+
+* ``{{ host_prefix }}`` - the pywb server origin, e.g. ``http://localhost:8080``
+
+* ``{{ config }}`` - provides the contents of the ``config.yaml`` as a dictionary.
+
+* ``{{ ui }}`` - an optional ``ui`` dictionary from ``config.yaml``, if any.
+
+The default banner creates the UI dynamically in JavaScript using Vue in the ``frame_insert.html`` template.
+
+
+Custom Banner Template
+^^^^^^^^^^^^^^^^^^^^^^
+
+File: ``custom_banner.html``
+
+This template can be used to render a custom banner for frameless replay. It is blank by default.
+
+In frameless replay, the content of this template is injected into the ``head_insert.html`` template to render the banner.
+
+
+Head Insert Template
+^^^^^^^^^^^^^^^^^^^^
+
+File: ``head_insert.html``
+
+This template represents the HTML injected into every replay page to add support for client-side rewriting via ``wombat.js``.
+
+This template is part of the core pywb replay, and modifying this template is not recommended. 
+
+For customizing the banner, modify the ``banner.html`` (framed replay) or ``custom_banner.html`` (frameless replay) template instead.
+
+
+Top Frame Template
+^^^^^^^^^^^^^^^^^^
+
+File: ``frame_insert.html``
+
+This template represents the top-level frame that is inserted to render the replay in framed mode.
+
+By design, this template does *not* extend from the base template.
+
+This template is responsible for creating the iframe that will render the content.
+
+This template only renders the banner and is designed *not* to set the encoding to allow the browser to 'detect' the encoding for the containing iframe.
+For this reason, the template should only contain ASCII text, and %-encode any non-ASCII characters.
+
+Content such as analytics code that is desired in the top frame of framed replay pages should be added to this template.
+
+Template variables:
+
+* ``{{ url }}`` - the URL being replayed.
+
+* ``{{ timestamp }}`` - the timestamp being replayed, e.g. ``20211226`` in ``http://localhost:8080/pywb/20211226/mp_/https://example.com/``
+
+* ``{{ wb_url }}`` - A complete ``WbUrl`` object, which contains the ``url``, ``timestamp`` and ``mod`` properties, representing the replay url.
+
+* ``{{ wb_prefix }}`` - the collection prefix, e.g. ``http://localhost:8080/pywb/``
+
+* ``{{ is_proxy }}`` - set to true if page is being loaded via an HTTP/S proxy (checks if WSGI env has ``wsgiprox.proxy_host`` set)
+
+* ``{{ ui }}`` - an optional ``ui`` dictionary from ``config.yaml``, if any.
+
+
+.. _custom-top-frame:
+
+Customizing the Top Frame Template
+""""""""""""""""""""""""""""""""""
+
+The top-frame used for framed replay can be replaced or augmented
+by modifying the ``frame_insert.html``.
+
+To start with modifying the default outer page, you can add it to the current
+templates directory by running ``wb-manager template --add frame_insert_html``
+
+To initialize the replay, the outer page should include ``wb_frame.js``,
+create an ``<iframe>`` element and pass the id (or element itself) to the ``ContentFrame`` constructor:
+
+.. code-block:: html
+
+  <script src='{{ host_prefix }}/{{ static_path }}/wb_frame.js'> </script>
+  <script>
+  var cframe = new ContentFrame({"url": "{{ url }}" + window.location.hash,
+                                 "prefix": "{{ wb_prefix }}",
+                                 "request_ts": "{{ wb_url.timestamp }}",
+                                 "iframe": "#replay_iframe"});
+  </script>
+
+
+The outer frame can receive notifications of changes to the replay via ``postMessage``
+
+For example, to detect when the content frame changed and log the new url and timestamp,
+use the following script in the outer frame html:
+
+.. code-block:: javascript
+
+  window.addEventListener("message", function(event) {
+    if (event.data.wb_type == "load" || event.data.wb_type == "replace-url") {
+      console.log("New Url: " + event.data.url);
+      console.log("New Timestamp: " + event.data.ts);
+    }
+  });
+
+The ``load`` message is sent when a new page is first loaded, while ``replace-url`` is used
+for url changes caused by content frame History navigation.
+
+
+Error Templates
+---------------
+
+The following templates are used to render errors.
+
+
+Page Not Found Template
+^^^^^^^^^^^^^^^^^^^^^^^
+
+File: ``not_found.html`` - template for 404 error pages.
+
+This template is used to render any 404/page not found errors that can occur when loading a URL that is not in the web archive.
+
+Template variables:
+
+* ``{{ url }}`` - the URL of the page
+
+* ``{{ wbrequest }}`` - the full ``WbRequest`` object which can be used to get additional info about the request.
+
+
+(The default template checks ``{{ wbrequest and wbrequest.env.pywb_proxy_magic }}`` to determine if the request is via an :ref:`https-proxy` connection or a regular request).
+
+
+Generic Error Template
+^^^^^^^^^^^^^^^^^^^^^^
+
+File: ``error.html`` - generic error template.
+
+
+This template is used to render all other errors that are not 'page not found'.
+
+Template variables:
+
+*  ``{{ err_msg }}`` - a shorter error message indicating what went wrong.
+
+*  ``{{ err_details }}`` - additional details about the error.
+
+
+
+
+.. _shared-template-vars:
+
+Shared Template Variables
+-------------------------
+
+The following template variables are available to all templates.
+
+* ``{{ env }}`` - contains environment variables passed to pywb.
+
+* ``{{ env.pywb_proxy_magic }}`` - if set, indicates pywb is accessed via proxy. See :ref:`https-proxy`
+
+* ``{{ static_prefix }}`` - URL path to use for loading static files.
+
+
+UI Configuration
+^^^^^^^^^^^^^^^^
+
+Starting with pywb 2.7.0, the ``ui`` block in ``config.yaml`` can contain any custom ui-specific settings.
+
+This block is provided to the ``search.html``, ``query.html`` and ``banner.html`` templates.
+
+
+Localization Globals
+^^^^^^^^^^^^^^^^^^^^
+
+The Localization system (see: :ref:`localization`) adds several additional template globals, to facilitate listing available locales and getting URLs to switch locales, including:
+
+* ``{{ _Q() }}`` - a function used to mark certain text for localization, e.g. ``{{ _Q('localize this text') }}``
+
+* ``{{ env.pywb_lang }}`` - indicates current locale language code used for localization.
+
+* ``{{ locales }}`` - a list of all available locale language codes, used for iterating over all locales.
+
+* ``{{ get_locale_prefixes() }}`` - a function which returns the prefixes to use to switch locales.
+
+* ``{{ switch_locale() }}`` - a function used to render a URL to switch locale for the current page. Ex: ``<a href="{{ switch_locale(locale) }}">{{ locale }}</a>`` renders a link to switch to a specific locale.
+

docs/manual/ui-customization.rst

@@ -1,141 +1,9 @@
-.. _ui-customizations:
+UI Customization
+================
 
-UI Customizations
------------------
+.. toctree::
 
-pywb supports UI customizations, either for an entire archive,
-or per-collection. Jinja2 templates are used for rendering all views,
-and static files can also be added as needed.
+   ui-guide
+   vue-ui
+   template-guide
  
-Templates
-^^^^^^^^^
-
-Default templates, listed below, are found in the ``./pywb/templates/`` directory.
-
-Custom template files placed in the ``templates`` directory, either in the root or per collection, will override that template.
-
-To copy the default pywb template to the template directory using the cli tools, run:
-
-``wb-manager template --add search_html``
-
-The following page-level templates are available, corresponding to home page, collection page or search results:
-
- * ``index.html`` -- Home Page Template, used for ``http://my-archive.example.com/``
-
- * ``search.html`` -- Collection Template, used for each collection page ``http://my-archive.example.com/<coll name>/``
-
- * ``query.html`` -- Capture Query Page for a given url, used for ``http://my-archive.example.com/<coll name/*/<url>``
-
-Error Pages:
-
- * ``not_found.html`` -- Page to show when a url is not found in the archive
-
- * ``error.html`` -- Generic Error Page for any error (except not found)
-
-Replay and Banner templates:
-
- * ``frame_insert.html`` -- Top-frame for framed replay mode (not used with frameless mode)
-
- * ``head_insert.html`` -- Rewriting code injected into ``<head>`` of each replayed page.
-   This template includes the banner template and itself should generally not need to be modified.
-
- * ``banner.html`` -- The banner used for frameless replay. Can be set to blank to disable the banner.
-
-
-To customize the default pywb UI across multiple pages, the following generic templates
-can also be overriden:
-
-* ``base.html`` -- The base template used for non-replay related pages.
-
-* ``head.html`` -- Template containing content to be added to the ``<head>`` of the ``base`` template
-
-* ``header.html`` -- Template to be added as the first content of the ``<body>`` tag of the ``base`` template
-
-* ``footer.html`` -- Template for adding content as the "footer" of the ``<body>`` tag of the ``base`` template
-
-
-The ``base.html`` template also provides five blocks that can be supplied by templates that extend it.
-
-* ``title`` -- Block for supplying the title for the page
-
-* ``head`` -- Block for adding content to the ``<head>``, includes ``head.html`` template
-
-* ``header`` -- Block for adding content to the ``<body>`` before the ``body`` block, includes the ``header.html`` template
-
-* ``body`` -- Block for adding the primary content to template
-
-* ``footer`` -- Block for adding content to the ``<body>`` after the ``body`` block, includes the ``footer.html`` template
-
-Static Files
-^^^^^^^^^^^^
-
-The pywb server will automatically support static files placed under the following directories:
-
-* Files under the root ``static`` directory can be accessed via ``http://my-archive.example.com/static/<filename>``
-
-* Files under the per-collection ``./collections/<coll name>/static`` directory can be accessed via ``http://my-archive.example.com/static/_/<coll name>/<filename>``
-
-
-Custom Metadata
-^^^^^^^^^^^^^^^
-
-It is possible to also add custom metadata that will be available in the Jinja2 template.
-
-For dynamic collections, any fields placed under ``<coll_name>/metadata.yaml`` filed can be accessed
-
-via the ``{{ metadata }}`` variable.
-
-For example, if metadata file contains:
-
-.. ex-block:: yaml
-
-    somedata: value
-
-Accessing ``{{ metadata.somedata }}`` will resolve to ``value``
-
-The metadata can also be added via commandline: ``wb-manager metadata myCollection --set somedata=value]``
-
-
-
-The default collection UI template (search.html) currently lists all of the available metadata fields.
-
-
-Custom Outer Replay Frame
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The top-frame used for framed replay can be replaced or augmented
-by modifying the ``frame_insert.html``.
-
-To start with modifying the default outer page, you can add it to the current
-templates directory by running ``wb-manager template --add frame_insert_html``
-
-To initialize the replay, the outer page should include ``wb_frame.js``,
-create an ``<iframe>`` element and pass the id (or element itself) to the ``ContentFrame`` constructor:
-
-.. code-block:: html
-
-  <script src='{{ host_prefix }}/{{ static_path }}/wb_frame.js'> </script>
-  <script>
-  var cframe = new ContentFrame({"url": "{{ url }}" + window.location.hash,
-                                 "prefix": "{{ wb_prefix }}",
-                                 "request_ts": "{{ wb_url.timestamp }}",
-                                 "iframe": "#replay_iframe"});
-  </script>
-
-
-The outer frame can receive notifications of changes to the replay via ``postMessage``
-
-For example, to detect when the content frame changed and log the new url and timestamp,
-use the following script to the outer frame html:
-
-.. code-block:: javascript
-
-  window.addEventListener("message", function(event) {
-    if (event.data.wb_type == "load" || event.data.wb_type == "replace-url") {
-      console.log("New Url: " + event.data.url);
-      console.log("New Timestamp: " + event.data.ts);
-    }
-  });
-
-The ``load`` message is sent when a new page is first loaded, while ``replace-url`` is used
-for url changes caused by content frame History navigation.

docs/manual/ui-guide.rst

@@ -0,0 +1,91 @@
+.. _ui-customizations:
+
+Customization Guide
+===================
+
+Most aspects of the pywb user-interface can be customized by changing the default styles, or overriding the HTML templates.
+
+This guide covers a few different options for customizing the UI.
+
+
+New Vue-based UI
+----------------
+
+With pywb 2.7.0, pywb includes a brand new UI which includes a visual calendar mode and a histogram-based banner.
+
+See :ref:`vue-ui` for more information on how to enable this UI.
+
+
+Customizing UI Templates
+------------------------
+
+pywb renders HTML using the Jinja2 templating engine, loading default templates from the ``pywb/templates`` directory.
+
+If running from a custom directory, templates can be placed in the ``templates`` directory and will override the defaults.
+
+See :ref:`template-guide` for more details on customizing the templates.
+
+
+Static Files
+------------
+
+pywb will automatically support static files placed under the following directories:
+
+* Files under the root ``static`` directory: ``static/my-file.js`` can be accessed via ``http://localhost:8080/static/my-file.js``
+
+
+* Files under the per-collection directory: ``./collections/my-coll/static/my-file.js`` can be accessed via ``http://localhost:8080/static/_/my-coll/my-file.js``
+
+
+It is possible to change these settings via ``config.yaml``:
+
+* ``static_prefix`` - sets the URL path used in pywb to serve static content (default ``static``)
+
+* ``static_dir`` - sets the directory name used to read static files on disk (default ``static``)
+
+While pywb can serve static files, it is recommended to use an existing web server to serve static files, especially if already using it in production.
+
+For example, this can be done via nginx with:
+
+
+.. code:: text
+
+    location /wayback/static {
+        alias /pywb/pywb/static;
+    }
+
+
+Loading Custom Metadata
+-----------------------
+
+pywb includes a default mechanism for loading externally defined metadata, loaded from a per-collection ``metadata.yaml`` YAML file at runtime.
+
+See :ref:`custom-metadata` for more details.
+
+Additionally, the banner template has access to the contents of the ``config.yaml`` via the ``{{ config }}`` template variable,
+allowing for passing in arbitrary config information.
+
+For more dynamic loading of data, the banner and all of the templates can load additional data via JS ``fetch()`` calls.
+
+
+Embedding pywb in frames
+------------------------
+
+It should be possible to embed pywb replay itself as an iframe as needed.
+
+For customizing the top-level page and banner, see :ref:`custom-top-frame`.
+
+However, there may be other reasons to embed pywb in an iframe.
+
+This can be done simply by including something like:
+
+.. code:: html
+
+   <html>
+     <head>
+       <body>
+         <div>Embedding pywb replay</div>
+         <iframe style="width: 100%; height: 100%" src="http://localhost:8080/pywb/20130729195151/http://test@example.com/"></iframe>
+      </body>
+   </html>
+

docs/manual/usage.rst

@@ -95,8 +95,8 @@ add the WARC to a new collection and start pywb:
 
       docker pull webrecorder/pywb
       docker run -e INIT_COLLECTION=my-web-archive -v /pywb-data:/webarchive \
-         -v /path/to:/source webrecorder/pywb wb-manager add default /path/to/my_warc.warc.gz
-      docker run -p 8080:8080 -v /pywb-data/:/webarchive wayback
+         -v /path/to:/source webrecorder/pywb wb-manager add my-web-archive /source/my_warc.warc.gz
+      docker run -p 8080:8080 -v /pywb-data/:/webarchive webrecorder/pywb wayback
 
 This example is equivalent to the non-Docker example above.
 
@@ -114,6 +114,8 @@ Using Existing Web Archive Collections
 Existing archives of WARCs/ARCs files can be used with pywb with minimal amount of setup. By using ``wb-manager add``,
 WARC/ARC files will automatically be placed in the collection archive directory and indexed.
 
+In pywb 2.8.0 and later, preliminary support for WACZ files is also added with ``wb-manager add --unpack-wacz``. This will unpack the provided WACZ file, adding its WARCs and indices to the collection.
+
 By default ``wb-manager``, places new collections in ``collections/<coll name>`` subdirectory in the current working directory. To specify a different root directory, the ``wb-manager -d <dir>``. Other options can be set in the config file.
 
 If you have a large number of existing CDX index files, pywb will be able to read them as well after running through a simple conversion process.
@@ -154,20 +156,20 @@ To enable auto-indexing, run with ``wayback -a`` or ``wayback -a --auto-interval
 Creating a Web Archive
 ----------------------
 
-Using Webrecorder
-^^^^^^^^^^^^^^^^^
+Using ArchiveWeb.page
+^^^^^^^^^^^^^^^^^^^^^
 
-If you do not have a web archive to test, one easy way to create one is to use `Webrecorder <https://webrecorder.io>`_
+If you do not have a web archive to test, one easy way to create one is to use the `ArchiveWeb.page <https://archiveweb.page>`_ browser extension for Chrome and other Chromium-based browsers such as Brave Browser. ArchiveWeb.page records pages visited during an archiving session in the browser, and provides means of both replaying and downloading the archived items created.
 
-After recording, you can click **Stop** and then click `Download Collection` to receive a WARC (`.warc.gz`) file.
+Follow the instructions in `How To Create Web Archives with ArchiveWeb.page <https://archiveweb.page/en/usage/>`_. After recording, press **Stop** and then `download your collection <https://archiveweb.page/en/download/>`_ to receive a WARC (`.warc.gz`) file. If you choose to download your collection in the WACZ format, the WARC files can be found inside the zipped WACZ in the ``archive/`` directory.
 
-You can then use this with work with pywb.
+You can then use your WARCs to work with pywb.
 
 
 Using pywb Recorder
 ^^^^^^^^^^^^^^^^^^^
 
-The core recording functionality in Webrecorder is also part of :mod:`pywb`. If you want to create a WARC locally, this can be
+Recording functionality is also part of :mod:`pywb`. If you want to create a WARC locally, this can be
 done by directly recording into your pywb collection:
 
 1. Create a collection: ``wb-manager init my-web-archive`` (if you haven't already created a web archive collection)
@@ -180,6 +182,14 @@ In this configuration, the indexing happens every 10 seconds.. After 10 seconds,
 ``http://localhost:8080/my-web-archive/http://example.com/``
 
 
+Using Browsertrix
+^^^^^^^^^^^^^^^^^
+
+For a more automated browser-based web archiving experience, `Browsertrix <https://browsertrix.com/>`_ provides a web interface for configuring, scheduling, running, reviewing, and curating crawls of web content. Crawl activity is shown in a live screencast of the browsers used for crawling and all web archives created in Browsertrix can be easily downloaded from the application in the WACZ format.
+
+`Browsertrix Crawler <https://crawler.docs.browsertrix.com/>`_, which provides the underlying crawling functionality of Browsertrix, can also be run standalone in a Docker container on your local computer.
+
+
 HTTP/S Proxy Mode Access
 ------------------------
 
@@ -206,6 +216,21 @@ pywb uses the gevent coroutine library, and the default app will support many co
 
 For larger scale production deployments, running with `uwsgi <http://uwsgi-docs.readthedocs.io/>`_ server application is recommended. The ``uwsgi.ini`` script provided can be used to launch pywb with uwsgi. uwsgi can be scaled to multiple processes to support the necessary workload, and pywb must be run with the `Gevent Loop Engine <http://uwsgi-docs.readthedocs.io/en/latest/Gevent.html>`_. Nginx or Apache can be used as an additional frontend for uwsgi.
 
+It is recommended to install uwsgi and its dependencies in a Python virtual environment (virtualenv). Consult the uwsgi documentation for `virtualenv support <https://uwsgi-docs.readthedocs.io/en/latest/Python.html#virtualenv-support>`_ for details on how to specify the virtualenv to uwsgi.
+
+Installation of uswgi in a virtualenv will avoid known issues with installing uwsgi in some Debian-based OSes with Python 3.9+. As an example, in Ubuntu 22.04 with Python 3.10, it is recommended to install uwsgi like so: ::
+
+    sudo apt install -y python3-pip \
+        python3-dev \
+        build-essential \
+        libssl-dev \
+        libffi-dev \
+        python3-setuptools \
+        python3-venv
+    python3 -m venv pywbenv
+    source pywbenv/bin/activate
+    pip install wheel uwsgi pywb
+
 Although uwsgi does not provide a way to specify command line, all command line options can alternatively be configured via ``config.yaml``. See :ref:`configuring-pywb` for more info on available configuration options.
 
 Docker Deployment
@@ -218,13 +243,13 @@ The following will run pywb in Docker directly on port 80:
 
 .. code:: console
 
-      docker run -p 80:8080 -v /webarchive-data/:/webarchive
+      docker run -p 80:8080 -v /webarchive-data/:/webarchive webrecorder/pywb
 
 To run pywb in Docker behind a local nginx (as shown below), port 8081 should also be mapped:
 
 .. code:: console
 
-      docker run -p 8081:8081 -v /webarchive-data/:/webarchive
+      docker run -p 8081:8081 -v /webarchive-data/:/webarchive webrecorder/pywb
 
 
 See :ref:`getting-started-docker` for more info on using pywb with Docker.

docs/manual/vue-ui.rst

@@ -0,0 +1,126 @@
+.. _vue-ui:
+
+
+Vue-based UI
+================
+
+With 2.7.0, pywb introduces a new `Vue UI <https://vuejs.org/>`_ based system, which provides a more feature-rich representation of a web archive.
+
+
+Overview
+--------
+
+Calendar UI
+^^^^^^^^^^^
+
+The new calendar UI provides a histogram and a clickable calendar representation of a web archive.
+
+The calendar is rendered in place of the URL query page from versions before 2.7.0.
+
+.. image:: images/vue-cal.png
+  :width: 600
+  :alt: Calendar UI Screenshot
+
+
+Banner Replay UI
+^^^^^^^^^^^^^^^^
+
+The new banner histogram allows for zooming in on captures per year, month, week, and day.
+
+Navigation preserves the different levels. The full calendar UI is also available as a dropdown by clicking the calendar icon.
+
+The new banner should allow for faster navigation across multiple captures.
+
+.. image:: images/vue-banner.png
+  :width: 600
+  :alt: Calendar UI Screenshot
+
+
+Custom Logo
+^^^^^^^^^^^
+
+It is possible to configure a custom logo by setting ``ui.logo`` in ``config.yaml`` to a static file.
+
+If omitted, the standard pywb logo will be used by default.
+
+If set, the logo should point to a file in the static directory (default is ``static`` but can be changed via the ``static_dir`` config option).
+
+For example, to use the file ``./static/my-logo.png`` as the logo, set:
+
+.. code:: yaml
+
+  ui:
+    logo: my-logo.png
+
+
+Logo URL
+^^^^^^^^
+
+It is possible to configure the logo to link to any URL by setting ``ui.logo_home_url`` in ``config.yml`` to the URL of your choice.
+
+If omitted, the logo will not link to any page.
+
+For example, to have the logo redirect to ``https://example.com/web-archive-landing-page``, set:
+
+.. code:: yaml
+
+  ui:
+    logo_home_url: https://example.com/web-archive-landing-page
+
+
+Printing
+^^^^^^^^
+
+As of pywb 2.8, the replay header includes a print button that prints the contents of the replay iframe.
+
+This button can be disabled by setting ``ui.disable_printing`` in ``config.yaml`` to any value.
+
+For example:
+
+.. code:: yaml
+
+  ui:
+    disable_printing: true
+
+
+Banner Colors
+^^^^^^^^^^^^^
+
+It is possible to configure the background color, text color, and button outlines of the header by setting values in the ``ui`` section of ``config.yaml``.
+
+To customize the header background color, set ``ui.navbar_background_hex`` to the color's hex value, with the initial hash symbol (``#``) omitted. If omitted, ``#f8f9fa`` (Bootstrap 4's ``light``) will be used by default.
+
+For example, to use the color ``#cff3ff`` as the banner color, set:
+
+.. code:: yaml
+
+  ui:
+    navbar_background_hex: cff3ff
+
+The navbar text color can similarly be set using the ``ui.navbar_color_hex`` setting.
+
+The banner's buttons default to Bootstrap 4's ``btn-outline-dark``. To use light-outlined buttons instead, set ``ui.navbar_light_buttons`` equal to any value.
+
+
+Updating the Vue UI
+-------------------
+
+The UI is contained within the ``pywb/vueui`` directory.
+
+The Vue component sources can be found in ``pywb/vueui/src``.
+
+Updating the UI requires ``node`` and ``yarn``.
+
+To install and build, run:
+
+
+.. code:: console
+
+   cd pywb/vueui
+   yarn install
+   yarn build
+
+
+This will generate the output to ``pywb/static/vue/vueui.js`` which is loaded from the default templates when the Vue UI rendering is enabled.
+
+Additional styles for the banner are loaded from ``pywb/static/vue_banner.css``.

pywb/apps/frontendapp.py

@@ -1,7 +1,7 @@
 from gevent.monkey import patch_all; patch_all()
 
 from werkzeug.routing import Map, Rule, RequestRedirect, Submount
-from werkzeug.wsgi import pop_path_info
+from wsgiref.util import shift_path_info
 from six.moves.urllib.parse import urljoin, parse_qsl
 from six import iteritems
 from warcio.utils import to_native_str
@@ -107,6 +107,8 @@ class FrontEndApp(object):
 
         self.templates_dir = config.get('templates_dir', 'templates')
         self.static_dir = config.get('static_dir', 'static')
+        self.static_prefix = config.get('static_prefix', 'static')
+        self.default_locale = config.get('default_locale', '')
 
         metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}', 'metadata.yaml')
         self.metadata_cache = MetadataCache(metadata_templ)
@@ -118,8 +120,8 @@ class FrontEndApp(object):
         specific routes (proxy mode, record)
         """
         self.url_map = Map()
-        self.url_map.add(Rule('/static/_/<coll>/<path:filepath>', endpoint=self.serve_static))
-        self.url_map.add(Rule('/static/<path:filepath>', endpoint=self.serve_static))
+        self.url_map.add(Rule('/{0}/_/<coll>/<path:filepath>'.format(self.static_prefix), endpoint=self.serve_static))
+        self.url_map.add(Rule('/{0}/<path:filepath>'.format(self.static_prefix), endpoint=self.serve_static))
         self.url_map.add(Rule('/collinfo.json', endpoint=self.serve_listing))
 
         if self.is_valid_coll('$root'):
@@ -363,6 +365,9 @@ class FrontEndApp(object):
         else:
             coll_config['metadata'] = self.metadata_cache.load(coll) or {}
 
+        if 'ui' in self.warcserver.config:
+            coll_config['ui'] = self.warcserver.config['ui']
+
         return coll_config
 
     def serve_coll_page(self, environ, coll='$root'):
@@ -380,6 +385,7 @@ class FrontEndApp(object):
 
         coll_config = self.get_coll_config(coll)
         metadata = coll_config.get('metadata')
+        ui = coll_config.get('ui', {})
 
         view = BaseInsertView(self.rewriterapp.jinja_env, 'search.html')
 
@@ -391,7 +397,8 @@ class FrontEndApp(object):
                                         wb_prefix=wb_prefix,
                                         coll=coll,
                                         coll_config=coll_config,
-                                        metadata=metadata)
+                                        metadata=metadata,
+                                        ui=ui)
 
         return WbResponse.text_response(content, content_type='text/html; charset="utf-8"')
 
@@ -408,6 +415,14 @@ class FrontEndApp(object):
         # if coll == self.all_coll:
         #    coll = '*'
 
+        config = self.warcserver.get_coll_config(coll)
+        is_live = config.get("index") == "$live"
+
+        if is_live:
+            cache_control = "no-store, no-cache"
+        else:
+            cache_control = "max-age=86400, must-revalidate"
+
         cdx_url = base_url.format(coll=coll)
 
         if environ.get('QUERY_STRING'):
@@ -419,14 +434,19 @@ class FrontEndApp(object):
             cdx_url += 'limit=' + str(self.query_limit)
 
         try:
-            res = requests.get(cdx_url, stream=True)
+            headers = {}
+            for key in environ.keys():
+                if key.startswith("HTTP_X_"):
+                    headers[key[5:].replace("_", "-")] = environ[key]
+            res = requests.get(cdx_url, stream=True, headers=headers)
 
             status_line = '{} {}'.format(res.status_code, res.reason)
             content_type = res.headers.get('Content-Type')
 
             return WbResponse.bin_stream(StreamIter(res.raw),
                                          content_type=content_type,
-                                         status=status_line)
+                                         status=status_line,
+                                         headers=[("Cache-Control", cache_control)])
 
         except Exception as e:
             return WbResponse.text_response('Error: ' + str(e), status='400 Bad Request')
@@ -538,9 +558,9 @@ class FrontEndApp(object):
             return
 
         if coll != '$root':
-            pop_path_info(environ)
+            shift_path_info(environ)
             if record:
-                pop_path_info(environ)
+                shift_path_info(environ)
 
         paths = [self.warcserver.root_dir]
 
@@ -583,7 +603,7 @@ class FrontEndApp(object):
         and message.
 
         :param dict environ: The WSGI environment dictionary for the request
-        :param str err_type: The identifier for type of error that occured
+        :param str err_type: The identifier for type of error that occurred
         :param str url: The url of the archived page that was requested
         """
         raise AppPageNotFound(err_type, url)
@@ -649,8 +669,12 @@ class FrontEndApp(object):
 
             lang = args.pop('lang', '')
             if lang:
-                pop_path_info(environ)
+                shift_path_info(environ)
+
+            if lang:
                 environ['pywb_lang'] = lang
+            elif self.default_locale:
+                environ['pywb_lang'] = self.default_locale
 
             response = endpoint(environ, **args)
 

pywb/apps/rewriterapp.py

@@ -2,7 +2,7 @@ from io import BytesIO
 
 import requests
 from fakeredis import FakeStrictRedis
-from six.moves.urllib.parse import unquote, urlencode, urlsplit, urlunsplit
+from six.moves.urllib.parse import unquote, urlencode, urlsplit, urlunsplit, parse_qsl
 from warcio.bufferedreaders import BufferedReader
 from warcio.recordloader import ArcWarcRecordLoader
 from warcio.timeutils import http_date_to_timestamp, timestamp_to_http_date
@@ -64,7 +64,7 @@ class RewriterApp(object):
 
         if not jinja_env:
             jinja_env = JinjaEnv(globals={'static_path': 'static'},
-                                 extensions=['jinja2.ext.i18n', 'jinja2.ext.with_'])
+                                 extensions=['jinja2.ext.i18n'])
             jinja_env.jinja_env.install_null_translations()
 
         self.jinja_env = jinja_env
@@ -78,10 +78,11 @@ class RewriterApp(object):
         self.redirect_to_exact = config.get('redirect_to_exact')
 
         self.banner_view = BaseInsertView(self.jinja_env, self._html_templ('banner_html'))
+        self.custom_banner_view = BaseInsertView(self.jinja_env, self._html_templ('custom_banner_html'))
 
         self.head_insert_view = HeadInsertView(self.jinja_env,
                                                self._html_templ('head_insert_html'),
-                                               self.banner_view)
+                                               self.custom_banner_view)
 
         self.frame_insert_view = TopFrameView(self.jinja_env,
                                               self._html_templ('frame_insert_html'),
@@ -532,6 +533,7 @@ class RewriterApp(object):
                                                    coll=kwargs.get('coll', ''),
                                                    replay_mod=self.replay_mod,
                                                    metadata=kwargs.get('metadata', {}),
+                                                   ui=kwargs.get('ui', {}),
                                                    config=self.config))
 
         cookie_rewriter = None
@@ -801,8 +803,17 @@ class RewriterApp(object):
     def handle_query(self, environ, wb_url, kwargs, full_prefix):
         prefix = self.get_full_prefix(environ)
 
+        res = dict(parse_qsl(environ.get("QUERY_STRING")))
+        is_advanced = res.get("matchType", "exact") != "exact" or res.get("url", "").endswith("*")
+
+        # vue ui not supported for advanced search for now
+        ui = kwargs.get("ui", {})
+        if is_advanced:
+            ui["vue_calendar_ui"] = False
+
         params = dict(url=wb_url.url,
-                      prefix=prefix)
+                      prefix=prefix,
+                      ui=ui)
 
         return self.query_view.render_to_string(environ, **params)
 
@@ -832,7 +843,7 @@ class RewriterApp(object):
 
     def get_rel_prefix(self, environ):
         # return request.script_name
-        return environ.get('SCRIPT_NAME') + '/'
+        return environ.get('SCRIPT_NAME', '') + '/'
 
     def get_full_prefix(self, environ):
         return self.get_host_prefix(environ) + self.get_rel_prefix(environ)
@@ -909,7 +920,9 @@ class RewriterApp(object):
         pass
 
     def get_top_frame_params(self, wb_url, kwargs):
-        return {'metadata': kwargs.get('metadata', {})}
+        return {'metadata': kwargs.get('metadata', {}),
+                'ui': kwargs.get('ui', {})
+               }
 
     def handle_custom_response(self, environ, wb_url, full_prefix, host_prefix, kwargs):
         if self.is_framed_replay(wb_url):

pywb/apps/wbrequestresponse.py

@@ -212,6 +212,7 @@ class WbResponse(object):
         self.status_headers.replace_header('Access-Control-Allow-Methods', allowed_methods)
         self.status_headers.replace_header('Access-Control-Allow-Credentials', 'true')
         self.status_headers.replace_header('Access-Control-Max-Age', '1800')
+        self.status_headers.replace_header('Cross-Origin-Resource-Policy', 'cross-origin')
         return self
 
     def __repr__(self):

pywb/default_config.yaml

@@ -12,9 +12,15 @@ templates_dir: templates
 
 # Template HTML
 banner_html: banner.html
+custom_banner_html: custom_banner.html
 head_insert_html: head_insert.html
 frame_insert_html: frame_insert.html
 
+base_html: base.html
+header_html: header.html
+footer_html: footer.html
+head_html: head.html
+
 query_html: query.html
 search_html: search.html
 not_found_html: not_found.html
@@ -31,6 +37,7 @@ info_json: collinfo.json
 # HTML Templates List
 html_templates:
     - banner_html
+    - custom_banner_html
     - head_insert_html
     - frame_insert_html
 
@@ -39,6 +46,12 @@ html_templates:
     - not_found_html
 
     - home_html
+
+    - base_html
+    - header_html
+    - head_html
+    - footer_html
+
     - error_html
     - proxy_cert_download_html
     - proxy_select_html

pywb/indexer/cdxindexer.py

@@ -1,5 +1,9 @@
+import logging
 import os
 import sys
+import traceback
+
+import warcio
 
 # Use ujson if available
 try:
@@ -298,8 +302,11 @@ def write_multi_cdx_index(output, inputs, **options):
                 with open(fullpath, 'rb') as infile:
                     entry_iter = record_iter(infile)
 
-                    for entry in entry_iter:
-                        writer.write(entry, filename)
+                    try:
+                        for entry in entry_iter:
+                            writer.write(entry, filename)
+                    except warcio.exceptions.ArchiveLoadFailed:
+                        logging.error('Error while indexing file %s, %s',filename,traceback.format_exc())
 
         return writer
 
@@ -377,7 +384,7 @@ url timestamp { ... }
 
     output_help = """
 Output file or directory.
-- If directory, each input file is written to a seperate output file
+- If directory, each input file is written to a separate output file
   with a .cdx extension
 - If output is '-', output is written to stdout
 """

pywb/manager/aclmanager.py

@@ -102,11 +102,11 @@ class ACLManager(CollectionsManager):
 
         except IOError as io:
             if must_exist:
-                print('Error Occured: ' + str(io))
+                print('Error Occurred: ' + str(io))
             return False
 
         except Exception as e:
-            print('Error Occured: ' + str(e))
+            print('Error Occurred: ' + str(e))
             return False
 
     def save_acl(self, r=None):

pywb/manager/locmanager.py

@@ -2,10 +2,14 @@ import os
 import os.path
 import shutil
 
-from babel.messages.frontend import CommandLineInterface
+try:
+    from babel.messages.frontend import CommandLineInterface
 
-from translate.convert.po2csv import main as po2csv
-from translate.convert.csv2po import main as csv2po
+    from translate.convert.po2csv import main as po2csv
+    from translate.convert.csv2po import main as csv2po
+    loc_avail = True
+except:
+    loc_avail = False
 
 
 ROOT_DIR = 'i18n'

pywb/manager/manager.py

@@ -5,12 +5,16 @@ import logging
 import heapq
 import yaml
 import re
+import gzip
 import six
+import pathlib
 
 from distutils.util import strtobool
 from pkg_resources import resource_string, get_distribution
 
 from argparse import ArgumentParser, RawTextHelpFormatter
+from tempfile import mkdtemp, TemporaryDirectory
+from zipfile import ZipFile
 
 from pywb.utils.loaders import load_yaml_config
 from warcio.timeutils import timestamp20_now
@@ -47,6 +51,9 @@ directory structure expected by pywb
 
     COLLS_DIR = 'collections'
 
+    WARC_RX = re.compile(r'.*\.w?arc(\.gz)?$')
+    WACZ_RX = re.compile(r'.*\.wacz$')
+
     def __init__(self, coll_name, colls_dir=None, must_exist=True):
         colls_dir = colls_dir or self.COLLS_DIR
         self.default_config = load_yaml_config(DEFAULT_CONFIG)
@@ -115,19 +122,142 @@ directory structure expected by pywb
                    'To create a new collection, run\n\n{1} init {0}')
             raise IOError(msg.format(self.coll_name, sys.argv[0]))
 
-    def add_warcs(self, warcs):
+    def add_archives(self, archives, unpack_wacz=False):
         if not os.path.isdir(self.archive_dir):
             raise IOError('Directory {0} does not exist'.
                           format(self.archive_dir))
 
-        full_paths = []
-        for filename in warcs:
-            filename = os.path.abspath(filename)
-            shutil.copy2(filename, self.archive_dir)
-            full_paths.append(os.path.join(self.archive_dir, filename))
-            logging.info('Copied ' + filename + ' to ' + self.archive_dir)
+        invalid_archives = []
+        warc_paths = []
+        for archive in archives:
+            if self.WARC_RX.match(archive):
+                full_path = self._add_warc(archive)
+                if full_path:
+                    warc_paths.append(full_path)
+            elif self.WACZ_RX.match(archive):
+                if unpack_wacz:
+                    self._add_wacz_unpacked(archive)
+                else:
+                    raise NotImplementedError('Adding waczs without unpacking is not yet implemented. Use '
+                                              '\'--unpack-wacz\' flag to add the wacz\'s content.')
+            else:
+                invalid_archives.append(archive)
 
-        self._index_merge_warcs(full_paths, self.DEF_INDEX_FILE)
+        self._index_merge_warcs(warc_paths, self.DEF_INDEX_FILE)
+
+        if invalid_archives:
+            logging.warning(f'Invalid archives weren\'t added: {", ".join(invalid_archives)}')
+
+    def _rename_warc(self, warc_basename):
+        dupe_idx = 1
+        ext = ''.join(pathlib.Path(warc_basename).suffixes)
+        pre_ext_name = warc_basename.split(ext)[0]
+
+        while True:
+            new_basename = f'{pre_ext_name}-{dupe_idx}{ext}'
+            if not os.path.exists(os.path.join(self.archive_dir, new_basename)):
+                break
+            dupe_idx += 1
+
+        return new_basename
+
+    def _add_warc(self, warc):
+        warc_source = os.path.abspath(warc)
+        source_dir, warc_basename = os.path.split(warc_source)
+
+        # don't overwrite existing warcs with duplicate names
+        if os.path.exists(os.path.join(self.archive_dir, warc_basename)):
+            warc_basename = self._rename_warc(warc_basename)
+            logging.info(f'Warc {os.path.basename(warc)} already exists - renamed to {warc_basename}.')
+
+        warc_dest = os.path.join(self.archive_dir, warc_basename)
+        shutil.copy2(warc_source, warc_dest)
+        logging.info(f'Copied {warc} to {self.archive_dir} as {warc_basename}')
+        return warc_dest
+
+    def _add_wacz_unpacked(self, wacz):
+        wacz = os.path.abspath(wacz)
+        temp_dir = mkdtemp()
+        warc_regex = re.compile(r'.+\.warc(\.gz)?$')
+        cdx_regex = re.compile(r'.+\.cdx(\.gz)?$')
+        with ZipFile(wacz, 'r') as wacz_zip_file:
+            archive_members = wacz_zip_file.namelist()
+            warc_files = [file for file in archive_members if warc_regex.match(file)]
+            if not warc_files:
+                logging.warning(f'WACZ {wacz} does not contain any warc files.')
+                return
+
+            # extract warc files
+            for warc_file in warc_files:
+                wacz_zip_file.extract(warc_file, temp_dir)
+
+            cdx_files = [file for file in archive_members if cdx_regex.match(file)]
+            if not cdx_files:
+                logging.warning(f'WACZ {wacz} does not contain any indices.')
+                return
+
+            for cdx_file in cdx_files:
+                wacz_zip_file.extract(cdx_file, temp_dir)
+
+        # copy extracted warc files to collections archive dir, use wacz filename as filename with added index if
+        # multiple warc files exist
+        warc_filename_mapping = {}
+        full_paths = []
+        for idx, extracted_warc_file in enumerate(warc_files):
+            _, warc_ext = os.path.splitext(extracted_warc_file)
+            if warc_ext == '.gz':
+                warc_ext = '.warc.gz'
+            warc_filename = os.path.basename(wacz)
+            warc_filename, _ = os.path.splitext(warc_filename)
+            warc_filename = f'{warc_filename}-{idx}{warc_ext}'
+            warc_destination_path = os.path.join(self.archive_dir, warc_filename)
+
+            if os.path.exists(warc_destination_path):
+                warc_filename = self._rename_warc(warc_filename)
+                logging.info(f'Warc {warc_destination_path} already exists - renamed to {warc_filename}.')
+                warc_destination_path = os.path.join(self.archive_dir, warc_filename)
+
+            warc_filename_mapping[os.path.basename(extracted_warc_file)] = warc_filename
+            shutil.copy2(os.path.join(temp_dir, extracted_warc_file), warc_destination_path)
+            full_paths.append(warc_destination_path)
+
+        # rewrite filenames in wacz indices and merge them with collection index file
+        for cdx_file in cdx_files:
+            self._add_wacz_index(os.path.join(self.indexes_dir, self.DEF_INDEX_FILE), os.path.join(temp_dir, cdx_file),
+                                 warc_filename_mapping)
+
+        # delete temporary files
+        shutil.rmtree(temp_dir)
+
+    def _add_wacz_index(self, collection_index_path, wacz_index_path, filename_mapping):
+        from pywb.warcserver.index.cdxobject import CDXObject
+
+        # rewrite wacz index to temporary index file
+        tempdir = TemporaryDirectory()
+        wacz_index_name = os.path.basename(wacz_index_path)
+        rewritten_index_path = os.path.join(tempdir.name, wacz_index_name)
+
+        with open(rewritten_index_path, 'w') as rewritten_index:
+            if wacz_index_path.endswith('.gz'):
+                wacz_index = gzip.open(wacz_index_path, 'rb')
+            else:
+                wacz_index = open(wacz_index_path, 'rb')
+
+            for line in wacz_index:
+                cdx_object = CDXObject(cdxline=line)
+                if cdx_object['filename'] in filename_mapping:
+                    cdx_object['filename'] = filename_mapping[cdx_object['filename']]
+                rewritten_index.write(cdx_object.to_cdxj())
+
+        if not os.path.isfile(collection_index_path):
+            shutil.move(rewritten_index_path, collection_index_path)
+            return
+
+        temp_coll_index_path = collection_index_path + '.tmp.' + timestamp20_now()
+        self._merge_indices(collection_index_path, rewritten_index_path, temp_coll_index_path)
+        shutil.move(temp_coll_index_path, collection_index_path)
+
+        tempdir.cleanup()
 
     def reindex(self):
         cdx_file = os.path.join(self.indexes_dir, self.DEF_INDEX_FILE)
@@ -180,20 +310,24 @@ directory structure expected by pywb
 
         merged_file = temp_file + '.merged'
 
-        last_line = None
-
-        with open(cdx_file, 'rb') as orig_index:
-            with open(temp_file, 'rb') as new_index:
-                with open(merged_file, 'w+b') as merged:
-                    for line in heapq.merge(orig_index, new_index):
-                        if last_line != line:
-                            merged.write(line)
-                            last_line = line
+        self._merge_indices(cdx_file, temp_file, merged_file)
 
         shutil.move(merged_file, cdx_file)
         #os.rename(merged_file, cdx_file)
         os.remove(temp_file)
 
+    @staticmethod
+    def _merge_indices(index1, index2, dest):
+        last_line = None
+
+        with open(index1, 'rb') as index1_f:
+            with open(index2, 'rb') as index2_f:
+                with open(dest, 'wb') as dest_f:
+                    for line in heapq.merge(index1_f, index2_f):
+                        if last_line != line:
+                            dest_f.write(line)
+                            last_line = line
+
     def set_metadata(self, namevalue_pairs):
         metadata_yaml = os.path.join(self.curr_coll_dir, 'metadata.yaml')
         metadata = None
@@ -237,17 +371,20 @@ directory structure expected by pywb
             v = defaults[n]
             print('- {0}: (pywb/{1})'.format(n, v))
 
-    def _confirm_overwrite(self, full_path, msg):
+    def _confirm_overwrite(self, full_path, msg, ignore=False):
         if not os.path.isfile(full_path):
             return True
 
+        if ignore:
+            return False
+
         res = get_input(msg)
         try:
             res = strtobool(res)
         except ValueError:
             res = False
 
-        if not res:
+        if not res and not ignore:
             raise IOError('Skipping, {0} already exists'.format(full_path))
 
     def _get_template_path(self, template_name, verb):
@@ -268,7 +405,7 @@ directory structure expected by pywb
 
         return full_path, filename
 
-    def add_template(self, template_name, force=False):
+    def add_template(self, template_name, force=False, ignore=False):
         full_path, filename = self._get_template_path(template_name, 'add')
 
         msg = ('Template file "{0}" ({1}) already exists. ' +
@@ -276,7 +413,11 @@ directory structure expected by pywb
         msg = msg.format(full_path, template_name)
 
         if not force:
-            self._confirm_overwrite(full_path, msg)
+            res = self._confirm_overwrite(full_path, msg, ignore)
+            if ignore and not res:
+                return
+
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
 
         data = resource_string('pywb', filename)
         with open(full_path, 'w+b') as fh:
@@ -286,6 +427,9 @@ directory structure expected by pywb
         msg = 'Copied default template "{0}" to "{1}"'
         print(msg.format(filename, full_path))
 
+        if template_name != "base_html":
+            self.add_template("base_html", force=False, ignore=True)
+
     def remove_template(self, template_name, force=False):
         full_path, filename = self._get_template_path(template_name, 'remove')
 
@@ -363,16 +507,23 @@ Create manage file based web archive collections
     listcmd = subparsers.add_parser('list', help=list_help)
     listcmd.set_defaults(func=do_list)
 
-    # Add Warcs
+    # Add Warcs or Waczs
     def do_add(r):
         m = CollectionsManager(r.coll_name)
-        m.add_warcs(r.files)
+        m.add_archives(r.files, r.unpack_wacz)
 
-    addwarc_help = 'Copy ARCS/WARCS to collection directory and reindex'
-    addwarc = subparsers.add_parser('add', help=addwarc_help)
-    addwarc.add_argument('coll_name')
-    addwarc.add_argument('files', nargs='+')
-    addwarc.set_defaults(func=do_add)
+    add_archives_help = 'Copy ARCs/WARCs to collection directory and reindex'
+    add_unpack_wacz_help = 'Copy WARCs from WACZ to collection directory and reindex'
+    add_archives = subparsers.add_parser('add', help=add_archives_help)
+    add_archives.add_argument(
+        '--unpack-wacz',
+        dest='unpack_wacz',
+        action='store_true',
+        help=add_unpack_wacz_help
+    )
+    add_archives.add_argument('coll_name')
+    add_archives.add_argument('files', nargs='+')
+    add_archives.set_defaults(func=do_add)
 
     # Reindex All
     def do_reindex(r):
@@ -448,12 +599,7 @@ Create manage file based web archive collections
     acl.set_defaults(func=do_acl)
 
     # LOC
-    loc_avail = False
-    try:
-        from pywb.manager.locmanager import LocManager
-        loc_avail = True
-    except:
-        pass
+    from pywb.manager.locmanager import LocManager, loc_avail
 
     def do_loc(r):
         if not loc_avail:

pywb/rewrite/content_rewriter.py

@@ -391,7 +391,7 @@ class StreamingRewriter(object):
 # ============================================================================
 class RewriteInfo(object):
     TAG_REGEX = re.compile(b'^(\xef\xbb\xbf)?\s*\<')
-    TAG_REGEX2 = re.compile(b'^.*<\w+[\s>]')
+    TAG_REGEX2 = re.compile(b'^.*<[!]?\w+[\s>]')
     JSON_REGEX = re.compile(b'^\s*[{[][{"]')  # if it starts with this then highly likely not HTML
 
     JSONP_CONTAINS = ['callback=jQuery',
@@ -524,7 +524,7 @@ class RewriteInfo(object):
         if not self.text_type:
             return False
 
-        if self.url_rewriter.wburl.mod == 'id_':
+        if self.is_identity():
             return False
 
         if self.url_rewriter.rewrite_opts.get('is_ajax'):
@@ -537,9 +537,11 @@ class RewriteInfo(object):
 
         return True
 
+    def is_identity(self):
+        return self.url_rewriter.wburl.mod in ('id_', 'ir_')
+
     def is_url_rw(self):
         if self.url_rewriter.wburl.mod in ('id_', 'bn_', 'wkrf_'):
             return False
 
         return True
-

pywb/rewrite/default_rewriter.py

@@ -20,7 +20,7 @@ from pywb.rewrite.rewrite_js_workers import JSWorkerRewriter
 from pywb import DEFAULT_RULES_FILE
 
 import copy
-from werkzeug.useragents import UserAgent
+from ua_parser import user_agent_parser
 
 
 # ============================================================================
@@ -34,7 +34,7 @@ class DefaultRewriter(BaseContentRewriter):
 
         'css': CSSRewriter,
 
-        'js': JSLocationOnlyRewriter,
+        'js': JSWombatProxyRewriter,
         'js-proxy': JSNoneRewriter,
         'js-worker': JSWorkerRewriter,
 
@@ -102,6 +102,7 @@ class DefaultRewriter(BaseContentRewriter):
         super(DefaultRewriter, self).__init__(rules_file, replay_mod)
         self.all_rewriters = copy.copy(self.DEFAULT_REWRITERS)
 
+        self.add_prefer_mod('raw', 'ir_')
         self.add_prefer_mod('raw', 'id_')
         self.add_prefer_mod('banner-only', 'bn_')
         self.add_prefer_mod('rewritten', replay_mod)
@@ -119,33 +120,44 @@ class RewriterWithJSProxy(DefaultRewriter):
         super(RewriterWithJSProxy, self).__init__(*args, **kwargs)
 
     def get_rewriter(self, rw_type, rwinfo=None):
-        if rw_type == 'js' and rwinfo:
-            # check if UA allows this
-            if self.ua_allows_obj_proxy(rwinfo.url_rewriter.rewrite_opts):
-                return JSWombatProxyRewriter
+        if rw_type != 'js' or not rwinfo:
+            return super(RewriterWithJSProxy, self).get_rewriter(rw_type, rwinfo)
 
-        # otherwise, return default rewriter
-        return super(RewriterWithJSProxy, self).get_rewriter(rw_type, rwinfo)
+        # check if should use old non-proxy rewriter
+        if self.ua_no_obj_proxy(rwinfo.url_rewriter.rewrite_opts):
+            print("loc only")
+            return JSLocationOnlyRewriter
+        else:
+        # otherwise, return default, js proxy-capable rewriter
+            return JSWombatProxyRewriter
 
-    def ua_allows_obj_proxy(self, opts):
+    def ua_no_obj_proxy(self, opts):
         ua = opts.get('ua')
         if not ua:
             ua_string = opts.get('ua_string')
             if ua_string:
-                ua = UserAgent(ua_string)
+                ua = user_agent_parser.ParseUserAgent(ua_string)
 
         if ua is None:
-            return True
+            return False
 
         supported = {
-            'chrome': '49.0',
-            'firefox': '44.0',
-            'safari': '10.0',
-            'opera': '36.0',
-            'edge': '12.0',
-            'msie': None,
+            'chrome': 49,
+            'firefox': 4,
+            'safari': 10,
+            'opera': 36,
+            'edge': 12,
+            'ie': 1000,
         }
 
-        min_vers = supported.get(ua.browser)
+        min_vers = supported.get(ua.get("family", "").lower())
+        if not min_vers:
+            return False
+
+        try:
+            ua_version = int(ua.get("major", 0))
+        except:
+            return False
+
+        return ua_version < min_vers
 
-        return (min_vers and ua.version >= min_vers)

pywb/rewrite/html_rewriter.py

@@ -177,7 +177,7 @@ class HTMLRewriterMixin(StreamingRewriter):
             return ''
 
         values = (url.strip() for url in re.split(self.SRCSET_REGEX, value) if url)
-        values = [self._rewrite_url(v.strip()) for v in values]
+        values = [self._rewrite_url(v.split(' ')[0].strip()) + (' ' + ' '.join(v.split(' ')[1:])).rstrip() for v in values if v]
         return ', '.join(values)
 
     def _rewrite_meta_refresh(self, meta_refresh):
@@ -268,7 +268,7 @@ class HTMLRewriterMixin(StreamingRewriter):
         unesc_value = self.try_unescape(value)
         rewritten_value = self.url_rewriter.rewrite(unesc_value, mod, force_abs)
 
-        # if no rewriting has occured, ensure we return original, not reencoded value
+        # if no rewriting has occurred, ensure we return original, not reencoded value
         if rewritten_value == value:
             return orig_value
 
@@ -433,6 +433,12 @@ class HTMLRewriterMixin(StreamingRewriter):
                     # URL not skipped, likely src='js/....', forcing abs to make sure, cause PHP MIME(JS) === HTML
                     attr_value = self._rewrite_url(attr_value, rw_mod, True)
                     self._write_attr('__wb_orig_src', ov, empty_attr=None)
+            
+            elif attr_name == 'target':
+                target = attr_value
+                if target in ('_blank', '_parent', '_top'):
+                    attr_value = '___wb_replay_top_frame'
+
             else:
                 # rewrite url using tag handler
                 rw_mod = handler.get(attr_name)
@@ -662,7 +668,7 @@ class HTMLRewriter(HTMLRewriterMixin, HTMLParser):
         if self.parse_comments:
             #data = self._rewrite_script(data)
 
-            # Rewrite with seperate HTMLRewriter
+            # Rewrite with separate HTMLRewriter
             comment_rewriter = HTMLRewriter(self.url_rewriter,
                                             defmod=self.defmod)
 

pywb/rewrite/regex_rewriters.py

@@ -16,6 +16,19 @@ class RxRules(object):
     def replace_str(replacer, match='this'):
         return lambda x, _: x.replace(match, replacer)
 
+    @staticmethod
+    def replace_prefix_from(prefix, match):
+        def do_replace(x, _):
+            start = x.find(match)
+            if start == 0:
+                return prefix
+            if start > 0:
+                return x[:start] + prefix
+            return x
+
+        return do_replace
+
+
     @staticmethod
     def format(template):
         return lambda string, _: template.format(string)
@@ -42,7 +55,7 @@ class RxRules(object):
         regex_str = '|'.join(['(' + rx + ')' for rx, op, count in rules])
 
         # ensure it's not middle of a word, wrap in non-capture group
-        regex_str = '(?<!\w)(?:' + regex_str + ')'
+        regex_str = '(?:' + regex_str + ')'
 
         return re.compile(regex_str, re.M)
 
@@ -84,6 +97,8 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj;
 
         check_loc = '((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = '
 
+        eval_str = 'WB_wombat_runEval2((_______eval_arg, isGlobal) => { var ge = eval; return isGlobal ? ge(_______eval_arg) : eval(_______eval_arg); }).eval(this, (function() { return arguments })(),'
+
         self.local_objs = [
             'window',
             'self',
@@ -102,16 +117,14 @@ if (!self.__WB_pmw) {{ self.__WB_pmw = function(obj) {{ this.__WB_source = obj;
 
         rules = [
             # rewriting 'eval(...)' - invocation
-            (r'(?<!function\s)(?:^|[^,$])eval\s*\(', self.replace_str('WB_wombat_runEval(function _____evalIsEvil(_______eval_arg$$) { return eval(_______eval_arg$$); }.bind(this)).eval', 'eval'), 0),
+            (r'(?<!function)(?:\s|^)\beval\s*\(', self.replace_prefix_from(eval_str, 'eval'), 0),
             # rewriting 'x = eval' - no invocation
             (r'(?<=[=,])\s*\beval\b\s*(?![(:.$])', self.replace_str('self.eval', 'eval'), 0),
             (r'(?<=\.)postMessage\b\(', self.add_prefix('__WB_pmw(self).'), 0),
-            (r'(?<![$.])\s*location\b\s*[=]\s*(?![=])', self.add_suffix(check_loc), 0),
+            (r'(?<![$.])\s*\blocation\b\s*[=]\s*(?![=])', self.add_suffix(check_loc), 0),
             # rewriting 'return this'
             (r'\breturn\s+this\b\s*(?![.$])', self.replace_str(this_rw), 0),
-            # rewriting 'this.' special properties access on new line, with ; prepended
-            (r'(?<=[\n])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(';' + this_rw), 0),
-            # rewriting 'this.' special properties access, not on new line (no ;)
+            # rewriting 'this.' special properties access
             (r'(?<![$.])\s*this\b(?=(?:\.(?:{0})\b))'.format(prop_str), self.replace_str(this_rw), 0),
             # rewrite '= this' or ', this'
             (r'(?<=[=,])\s*this\b\s*(?![:.$])', self.replace_str(this_rw), 0),
@@ -344,7 +357,7 @@ class CSSRewriter(RegexRewriter):
 class XMLRules(RxRules):
     def __init__(self):
         rules = [
-            ('([A-Za-z:]+[\s=]+)?["\'\s]*(' +
+            ('(?<![\w])([A-Za-z:]+[\s=]+)?["\'\s]*(' +
              self.HTTPX_MATCH_STR + ')',
              self.archival_rewrite(), 2),
         ]

pywb/rewrite/rewrite_dash.py

@@ -90,17 +90,25 @@ def rewrite_tw_dash(string, *args):
     try:
         best_variant = None
         best_bitrate = 0
+        best_src = ""
         max_bitrate = 5000000
 
         data = json.loads(string)
         for variant in data["variants"]:
-            if variant["content_type"] != "video/mp4":
+            if (("content_type" in variant and variant["content_type"] != "video/mp4") or
+                ("type" in variant and variant["type"] != "video/mp4")):
                 continue
 
             bitrate = variant.get("bitrate")
+            src = variant.get("src")
+
             if bitrate and bitrate > best_bitrate and bitrate <= max_bitrate:
                 best_variant = variant
                 best_bitrate = bitrate
+            # just compare src strings with dimensions
+            elif src and src > best_src:
+                best_variant = variant
+                best_src = src
 
         if best_variant:
             data["variants"] = [best_variant]

pywb/rewrite/templateview.py

@@ -5,7 +5,7 @@ from pywb.utils.loaders import load
 
 from six.moves.urllib.parse import urlsplit, quote
 
-from jinja2 import Environment, TemplateNotFound, contextfunction, select_autoescape
+from jinja2 import Environment, TemplateNotFound, pass_context, select_autoescape
 from jinja2 import FileSystemLoader, PackageLoader, ChoiceLoader
 
 from webassets.ext.jinja2 import AssetsExtension
@@ -139,7 +139,7 @@ class JinjaEnv(object):
             return loc_map.get(loc)
 
         def override_func(jinja_env, name):
-            @contextfunction
+            @pass_context
             def get_override(context, text):
                 translate = get_translate(context)
                 if not translate:
@@ -158,7 +158,7 @@ class JinjaEnv(object):
 
         # Special _Q() function to return %-encoded text, necessary for use
         # with text in banner
-        @contextfunction
+        @pass_context
         def quote_gettext(context, text):
             translate = get_translate(context)
             if not translate:
@@ -171,14 +171,14 @@ class JinjaEnv(object):
         self.jinja_env.globals['_Q'] = quote_gettext
         self.jinja_env.globals['default_locale'] = default_locale
 
-        @contextfunction
+        @pass_context
         def switch_locale(context, locale):
             environ = context.get('env')
             curr_loc = environ.get('pywb_lang', '')
 
             request_uri = environ.get('REQUEST_URI', environ.get('PATH_INFO'))
 
-            if curr_loc:
+            if curr_loc and request_uri.startswith('/' + curr_loc + '/'):
                 return request_uri.replace(curr_loc, locale, 1)
 
             app_prefix = environ.get('pywb.app_prefix', '')
@@ -188,7 +188,7 @@ class JinjaEnv(object):
 
             return app_prefix + '/' + locale + request_uri
 
-        @contextfunction
+        @pass_context
         def get_locale_prefixes(context):
             environ = context.get('env')
             locale_prefixes = {}
@@ -196,11 +196,11 @@ class JinjaEnv(object):
             orig_prefix = environ.get('pywb.app_prefix', '')
             coll = environ.get('SCRIPT_NAME', '')
 
-            if orig_prefix:
+            if orig_prefix and coll.startswith(orig_prefix):
                 coll = coll[len(orig_prefix):]
 
             curr_loc = environ.get('pywb_lang', '')
-            if curr_loc:
+            if curr_loc and coll.startswith('/' + curr_loc):
                 coll = coll[len(curr_loc) + 1:]
 
             for locale in loc_map.keys():
@@ -371,7 +371,7 @@ class HeadInsertView(BaseInsertView):
 
             if self.banner_view:
                 banner_html = self.banner_view.render_to_string(env, cdx=cdx, **params)
-                params['banner_html'] = banner_html
+                params['custom_banner_html'] = banner_html
 
             return self.render_to_string(env, cdx=cdx, **params)
 
@@ -405,10 +405,11 @@ class TopFrameView(BaseInsertView):
 
         embed_url = wb_url.to_str(mod=replay_mod)
 
+        timestamp = ''
         if wb_url.timestamp:
             timestamp = wb_url.timestamp
-        else:
-            timestamp = timestamp_now()
+        #else:
+        #    timestamp = timestamp_now()
 
         is_proxy = 'wsgiprox.proxy_host' in env
 

pywb/rewrite/test/test_content_rewriter.py

@@ -13,7 +13,7 @@ from pywb.utils.io import chunk_encode_iter
 
 from pywb.rewrite.wburl import WbUrl
 from pywb.rewrite.url_rewriter import UrlRewriter
-from pywb.rewrite.default_rewriter import DefaultRewriter, RewriterWithJSProxy
+from pywb.rewrite.default_rewriter import RewriterWithJSProxy
 
 from pywb import get_test_dir
 
@@ -39,8 +39,7 @@ def headers(request):
 class TestContentRewriter(object):
     @classmethod
     def setup_class(self):
-        self.content_rewriter = DefaultRewriter()
-        self.js_proxy_content_rewriter = RewriterWithJSProxy()
+        self.content_rewriter = RewriterWithJSProxy()
 
     def _create_response_record(self, url, headers, payload, warc_headers):
         writer = BufferWARCWriter()
@@ -65,7 +64,6 @@ class TestContentRewriter(object):
         record = self._create_response_record(url, headers, content, warc_headers)
 
         wburl = WbUrl(ts + '/' + (request_url or url))
-        url_rewriter = UrlRewriter(wburl, prefix)
 
         cdx = CDXObject()
         cdx['url'] = url
@@ -79,11 +77,13 @@ class TestContentRewriter(object):
             return ''
 
         if use_js_proxy:
-            rewriter = self.js_proxy_content_rewriter
+            rewrite_opts = {}
         else:
-            rewriter = self.content_rewriter
+            rewrite_opts = {'ua_string': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/10.0 Safari/537.36'}
 
-        return rewriter(record, url_rewriter, cookie_rewriter=None,
+        url_rewriter = UrlRewriter(wburl, prefix, rewrite_opts=rewrite_opts)
+
+        return self.content_rewriter(record, url_rewriter, cookie_rewriter=None,
                         head_insert_func=insert_func,
                         cdx=cdx,
                         environ=environ)
@@ -141,6 +141,17 @@ class TestContentRewriter(object):
         assert ('Content-Type', 'text/html; charset=utf-8') in headers.headers
         assert b''.join(gen).decode('utf-8') == exp
 
+    def test_rewrite_html_ignore_bom(self):
+        headers = {'Content-Type': 'text/html'}
+        content = u'\ufeff\ufeff\ufeff<!DOCTYPE html>\n<head>\n<a href="http://example.com"></a></body></html>'
+
+        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701mp_')
+
+        exp = '\ufeff\ufeff\ufeff<!DOCTYPE html>\n<head>\n<a href="http://localhost:8080/prefix/201701/http://example.com"></a></body></html>'
+        assert is_rw
+        assert ('Content-Type', 'text/html') in headers.headers
+        assert b''.join(gen).decode('utf-8') == exp
+
     def test_rewrite_html_utf_8_anchor(self):
         headers = {'Content-Type': 'text/html; charset=utf-8'}
         content = u'<html><body><a href="#éxample-tésté"></a></body></html>'

pywb/rewrite/test/test_html_rewriter.py

@@ -185,6 +185,10 @@ r"""
 >>> parse('<img srcset="//example.com/1x,1x 2w, //example1.com/foo 2x, http://example.com/bar,bar 4x">')
 <img srcset="/web/20131226101010///example.com/1x,1x 2w, /web/20131226101010///example1.com/foo 2x, /web/20131226101010/http://example.com/bar,bar 4x">
 
+# complex srcset attrib
+>>> parse('<img srcset="http://test.com/yaşar-kunduz.jpg 320w, http://test.com/yaşar-konçalves-273x300.jpg 273w">')
+<img srcset="/web/20131226101010/http://test.com/ya%C5%9Far-kunduz.jpg 320w, /web/20131226101010/http://test.com/ya%C5%9Far-konc%CC%A7alves-273x300.jpg 273w">
+
 # empty srcset attrib
 >>> parse('<img srcset="">')
 <img srcset="">
@@ -412,6 +416,9 @@ r"""
 >>> parse('<!--[if !IE]> --><html><![endif]--><a href="http://example.com/"><!--[if IE]><![endif]--><a href="http://example.com/"></html>')
 <!--[if !IE]> --><html><![endif]><a href="/web/20131226101010/http://example.com/"><!--[if IE]><![endif]--><a href="/web/20131226101010/http://example.com/"></html>
 
+# Test tag with a target
+>>> parse('<HTML><A Href=\"page.html\" target=\"_blank\">Text</a></hTmL>')
+<html><a href="page.html" target="___wb_replay_top_frame">Text</a></html>
 
 # Test blank
 >>> parse('')

pywb/rewrite/test/test_regex_rewriters.py

@@ -143,7 +143,7 @@ r"""
 'var foo = _____WB$wombat$check$this$function_____(this).location'
 
 >>> _test_js_obj_proxy('A = B\nthis.location = "foo"')
-'A = B\n;_____WB$wombat$check$this$function_____(this).location = "foo"'
+'A = B\n_____WB$wombat$check$this$function_____(this).location = "foo"'
 
 >>> _test_js_obj_proxy('var foo = this.location2')
 'var foo = this.location2'
@@ -216,7 +216,12 @@ r"""
 'this. location = ((self.__WB_check_loc && self.__WB_check_loc(location, arguments)) || {}).href = http://example.com/'
 
 >>> _test_js_obj_proxy('eval(a)')
-'WB_wombat_runEval(function _____evalIsEvil(_______eval_arg$$) { return eval(_______eval_arg$$); }.bind(this)).eval(a)'
+'WB_wombat_runEval2((_______eval_arg, isGlobal) => { var ge = eval; return isGlobal ? ge(_______eval_arg) : eval(_______eval_arg); }).eval(this, (function() { return arguments })(),a)'
+
+>>> _test_js_obj_proxy('abc eval(a)')
+'abc WB_wombat_runEval2((_______eval_arg, isGlobal) => { var ge = eval; return isGlobal ? ge(_______eval_arg) : eval(_______eval_arg); }).eval(this, (function() { return arguments })(),a)'
+
+
 
 >>> _test_js_obj_proxy(',eval(a)')
 ',eval(a)'
@@ -234,7 +239,7 @@ r"""
 '$eval = self.eval; $eval(a);'
 
 >>> _test_js_obj_proxy('foo(a, eval(data));')
-'foo(a, WB_wombat_runEval(function _____evalIsEvil(_______eval_arg$$) { return eval(_______eval_arg$$); }.bind(this)).eval(data));'
+'foo(a, WB_wombat_runEval2((_______eval_arg, isGlobal) => { var ge = eval; return isGlobal ? ge(_______eval_arg) : eval(_______eval_arg); }).eval(this, (function() { return arguments })(),data));'
 
 >>> _test_js_obj_proxy('function eval() {}')
 'function eval() {}'
@@ -254,6 +259,12 @@ r"""
 >>> _test_js_obj_proxy('x = obj.eval(a)')
 'x = obj.eval(a)'
 
+>>> _test_js_obj_proxy('x = obj._eval(a)')
+'x = obj._eval(a)'
+
+>>> _test_js_obj_proxy('x = obj.$eval(a)')
+'x = obj.$eval(a)'
+
 
 #=================================================================
 # XML Rewriting
@@ -356,7 +367,6 @@ def _test_xml(string):
 def _test_css(string):
     return CSSRewriter(urlrewriter).rewrite(string)
 
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod()

pywb/rules.yaml

@@ -61,6 +61,7 @@ rules:
 
     # twitter rules
     #=================================================================
+
     - url_prefix: 'com,twitter)/i/profiles/show/'
 
       fuzzy_lookup: '/profiles/show/.*with_replies\?.*(max_id=[^&]+)'
@@ -84,6 +85,14 @@ rules:
               function: 'pywb.rewrite.rewrite_dash:rewrite_tw_dash'
 
  
+    - url_prefix: ['com,twimg,syndication,cdn)/tweet-result']
+
+      rewrite:
+        js_regexs:
+            - match: 'video":(.*?viewCount":\d+})'
+              group: 1
+              function: 'pywb.rewrite.rewrite_dash:rewrite_tw_dash'
+
      
 
     # facebook rules
@@ -101,7 +110,7 @@ rules:
 
       fuzzy_lookup:
         match: '("(?:cursor|cursorindex)":["\d\w]+)'
-        find_all: true
+        re_type: findall
 
     - url_prefix: 'com,facebook)/ajax/pagelet/generic.php/profiletimeline'
       fuzzy_lookup: 'com,facebook\)/.*[?&](__adt=[^&]+).*[&]data=(?:.*?(?:[&]|(profile_id|pagelet_token)[^,]+))'
@@ -166,7 +175,7 @@ rules:
 
       fuzzy_lookup:
         match: '("q[\d]+":|after:\\"[^"]+)'
-        find_all: true
+        re_type: findall
 
     - url_prefix: 'com,facebook)/pages_reaction_units/more'
 
@@ -187,6 +196,9 @@ rules:
               group: 1
               function: 'pywb.rewrite.rewrite_dash:rewrite_fb_dash'
 
+            - match: '"debugNoBatching\s?":(?:false|0)'
+              replace: '"debugNoBatching":true'
+
         parse_comments: true
 
     - url_prefix: 'com,facebook'
@@ -218,6 +230,9 @@ rules:
             - match: '"is_dash_eligible":true'
               replace: '"is_dash_eligible":false'
 
+            - match: '"debugNoBatching\s?":(?:false|0)'
+              replace: '"debugNoBatching":true'
+
       fuzzy_lookup: '()'
 
 
@@ -529,6 +544,12 @@ rules:
       rewrite:
         js_rewrite_location: urls
 
+    - url_prefix: 'com,example)/matched'
+      fuzzy_lookup:
+        re_type: sub
+        match: 'matched'
+        replace: 'replaced'          
+
     # all domain rules -- fallback to this dataset
     #=================================================================
     # Applies to all urls -- should be last

pywb/static/css/base.css

@@ -39,3 +39,7 @@ header .language-select a:hover {
     text-decoration: underline;
 }
 
+.error pre {
+    white-space: pre-wrap;
+    text-align: left;
+}

pywb/static/default_banner.css

@@ -1,191 +0,0 @@
-
-#_wb_frame_top_banner
-{
-    display: block !important;
-    top: 0px !important;
-    left: 0px !important;
-    font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif !important;
-    width: 100% !important;
-    font-size: 18px !important;
-    background-color: #444 !important;
-    color: white !important;
-    z-index: 2147483643 !important;
-    line-height: normal !important;
-
-    position: absolute !important;
-    border: 0px;
-    height: 44px !important;
-
-    display: flex !important;
-    display: -webkit-box !important;
-    display: -moz-box !important;
-    display: -webkit-flex !important;
-    display: -ms-flexbox !important;
-
-    justify-content: space-between;
-    -webkit-box-pack: justify;
-    -moz-box-pack: justify;
-    -ms-flex-pack: justify;
-    align-items: center;
-    -webkit-box-align: center;
-    -moz-box-align: center;
-    -ms-flex-align: center;
-}
-
-#title_or_url
-{
-    display: block !important;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-    max-width: 100%;
-}
-
-
-
-#_wb_frame_top_banner ._wb_linked_logo
-{
-    display: block;
-    height: 26px;
-    width: 71px;
-    margin-left: 15px;
-    flex-shrink: 0;
-    -webkit-flex-shrink: 1 0;
-    -moz-flex-shrink: 1 0;
-    -ms-flex: 0 0 71px;
-}
-
-#_wb_frame_top_banner ._wb_linked_logo img
-{
-    width: auto;
-    height: 100%;
-    border: none;
-}
-
-#_wb_capture_info
-{
-    flex-grow: 1;
-    -webkit-box-flex: 1;
-    -moz-box-flex: 1;
-    -webkit-flex-grow: 1;
-    -ms-flex: 1;
-
-    min-width: 0;
-    margin: 0 15px;
-
-    display: flex !important;
-    display: -webkit-box !important;
-    display: -moz-box !important;
-    display: -webkit-flex !important;
-    display: -ms-flexbox !important;
-
-    flex-direction: column;
-    -webkit-box-direction: normal;
-    -webkit-box-orient: vertical;
-    -moz-box-direction: normal;
-    -moz-box-orient: vertical;
-    -ms-flex-direction: column;
-
-    justify-content: center;
-    -webkit-box-pack: center;
-    -moz-box-pack: center;
-    -ms-flex-pack: center;
-
-    align-items: center;
-    -webkit-box-align: center;
-    -moz-box-align: center;
-    -ms-flex-align: center;
-
-    height: 100%;
-    -webkit-font-smoothing: antialiased;
-}
-
-._wb_capture_date
-{
-    font-size: 13px;
-}
-
-#_wb_frame_top_banner #_wb_ancillary_links
-{
-    font-size: 12px;
-    color: #FFF;
-    text-align: right;
-    margin: 0px 15px 0px 0px;
-    padding: inherit;
-    background-color: inherit;
-    width: initial;
-    flex-shrink: 1;
-    -webkit-flex-shrink: 1;
-    -moz-flex-shrink: 1;
-    -ms-flex: 0 0 115px;
-}
-#_wb_frame_top_banner #_wb_ancillary_links a:link,
-#_wb_frame_top_banner #_wb_ancillary_links a:visited,
-#_wb_frame_top_banner #_wb_ancillary_links a:active
-{
-    color: #FFF;
-    text-decoration: none;
-}
-#_wb_frame_top_banner #_wb_ancillary_links a:hover
-{
-    text-decoration: underline;
-}
-#_wb_frame_top_banner #_wb_ancillary_links a img
-{
-    width: 10px;
-    height: 10px;
-}
-
-#wb_iframe_div
-{
-    position: absolute;
-    width: 100%;
-    height: 100%;
-    padding: 44px 0px 0px 0px;
-    border: none;
-    box-sizing: border-box;
-    -moz-box-sizing: border-box;
-    -webkit-box-sizing: border-box;
-    overflow: hidden;
-}
-
-.wb_iframe
-{
-    width: 100%;
-    height: 100%;
-    border: 2px solid #FFF;
-    border-width: 2px 0 0 0;
-    padding: 0px 0px 0px 0px;
-    overflow: scroll;
-}
-
-._wb_mobile {
-    display: none;
-}
-
-@media screen and (max-width: 500px) {
-    #_wb_frame_top_banner ._wb_linked_logo
-    {
-        width: 26px;
-        height: 26px;
-        margin-left: 10px;
-    }
-    #_wb_frame_top_banner ._wb_linked_logo img:not(._wb_mobile)
-    {
-        display: none;
-    }
-    #_wb_frame_top_banner ._wb_mobile
-    {
-        display: block;
-    }
-
-    #_wb_capture_info
-    {
-        margin: 0 5px;
-    }
-
-    #_wb_frame_top_banner ._wb_no-mobile
-    {
-        display: none;
-    }
-}

pywb/static/default_banner.js

@@ -1,320 +0,0 @@
-/*
-Copyright(c) 2013-2018 Rhizome and Ilya Kreymer. Released under the GNU General Public License.
-
-This file is part of pywb, https://github.com/webrecorder/pywb
-
-    pywb is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    pywb is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with pywb.  If not, see <http://www.gnu.org/licenses/>.
-
-*/
-
-// Creates the default pywb banner.
-
-(function() {
-  if (window.top !== window) {
-    return;
-  }
-
-  /**
-   * The default banner class
-   */
-  function DefaultBanner() {
-    if (!(this instanceof DefaultBanner)) return new DefaultBanner();
-    this.banner = null;
-    this.captureInfo = null;
-    this.last_state = {};
-    this.state = null;
-    this.title = '';
-    this.bannerUrlSet = false;
-    this.onMessage = this.onMessage.bind(this);
-  }
-
-  // Functions required to be exposed by all banners
-
-  /**
-   * @desc Initialize (display) the banner
-   */
-  DefaultBanner.prototype.init = function() {
-    this.createBanner('_wb_frame_top_banner');
-
-    if (window.wbinfo) {
-      this.set_banner(
-        window.wbinfo.url,
-        window.wbinfo.timestamp,
-        window.wbinfo.is_live,
-        window.wbinfo.is_framed ? '' : document.title
-      );
-    }
-  };
-
-  /**
-   * @desc Called by ContentFrame to detect if the banner is still showing
-   * that the page is loading
-   * @returns {boolean}
-   */
-  DefaultBanner.prototype.stillIndicatesLoading = function() {
-    return !this.bannerUrlSet;
-  };
-
-  /**
-   * @param {string} url - The URL of the replayed page
-   * @param {?string} ts - The timestamp of the replayed page.
-   * If we are in live mode this is undefined/empty string
-   * @param {boolean} is_live - A bool indicating if we are operating in live mode
-   */
-  DefaultBanner.prototype.updateCaptureInfo = function(url, ts, is_live) {
-    if (is_live && !ts) {
-      ts = new Date().toISOString().replace(/[-T:.Z]/g, '');
-    }
-    this.set_banner(url, ts, is_live, null);
-  };
-
-  /**
-   * @desc Called by ContentFrame when a message is received from the replay iframe
-   * @param {MessageEvent} event - The message event containing the message received
-   * from the replayed page
-   */
-  DefaultBanner.prototype.onMessage = function(event) {
-    var type = event.data.wb_type;
-
-    if (type === 'load' || type === 'replace-url') {
-      this.state = event.data;
-      this.last_state = this.state;
-      this.title = event.data.title || this.title;
-    } else if (type === 'title') {
-      this.state = this.last_state;
-      this.title = event.data.title;
-    } else {
-      return;
-    }
-
-    // favicon update
-    if (type === 'load') {
-      var head = document.querySelector('head');
-      var oldLink = document.querySelectorAll("link[rel*='icon']");
-      var i = 0;
-      for (; i < oldLink.length; i++) {
-        head.removeChild(oldLink[i]);
-      }
-
-      if (this.state.icons) {
-        for (i = 0; i < this.state.icons.length; i++) {
-          var icon = this.state.icons[i];
-          var link = document.createElement('link');
-          link.rel = icon.rel;
-          link.href = icon.href;
-          head.appendChild(link);
-        }
-      }
-    }
-
-    this.set_banner(
-      this.state.url,
-      this.state.ts,
-      this.state.is_live,
-      this.title
-    );
-  };
-
-  // Functions internal to the default banner
-
-  /**
-   * @desc Navigate to different language, if available
-   */
-
-  DefaultBanner.prototype.changeLanguage = function(lang, evt) {
-    evt.preventDefault();
-    var path = window.location.href;
-    if (path.indexOf(window.banner_info.prefix) == 0) {
-      path = path.substring(window.banner_info.prefix.length);
-      if (window.banner_info.locale_prefixes && window.banner_info.locale_prefixes[lang]) {
-        window.location.pathname = window.banner_info.locale_prefixes[lang] + path;
-      }
-    }
-  }
-
-  /**
-   * @desc Creates the underlying HTML elements comprising the banner
-   * @param {string} bid - The id for the banner
-   */
-  DefaultBanner.prototype.createBanner = function(bid) {
-    this.banner = document.createElement('wb_div', true);
-    this.banner.setAttribute('id', bid);
-    this.banner.setAttribute('lang', 'en');
-
-    if (window.banner_info.logoImg) {
-      var logo = document.createElement("a");
-      logo.setAttribute("href", "/" + (window.banner_info.locale ? window.banner_info.locale + "/" : ""));
-      logo.setAttribute("class", "_wb_linked_logo");
-
-      var logoContents = "";
-      logoContents += "<img src='" + window.banner_info.logoImg + "' alt='" + window.banner_info.logoAlt + "'>";
-      logoContents += "<img src='" + window.banner_info.logoImg + "' class='_wb_mobile' alt='" + window.banner_info.logoAlt + "'>";
-
-      logo.innerHTML = logoContents;
-      this.banner.appendChild(logo);
-    }
-
-    this.captureInfo = document.createElement("span");
-    this.captureInfo.setAttribute("id", "_wb_capture_info");
-    this.captureInfo.innerHTML = window.banner_info.loadingLabel;
-    this.banner.appendChild(this.captureInfo);
-
-    var ancillaryLinks = document.createElement("div");
-    ancillaryLinks.setAttribute("id", "_wb_ancillary_links");
-
-    var calendarImg = window.banner_info.calendarImg || window.banner_info.staticPrefix + "/calendar.svg";
-
-    var calendarLink = document.createElement("a");
-    calendarLink.setAttribute("id", "calendarLink");
-    calendarLink.setAttribute("href", "#");
-    calendarLink.innerHTML = "<img src='" + calendarImg + "' alt='" + window.banner_info.calendarAlt + "'><span class='_wb_no-mobile'>&nbsp;" +window.banner_info.calendarLabel + "</span>";
-    ancillaryLinks.appendChild(calendarLink);
-    this.calendarLink = calendarLink;
-
-    if (typeof window.banner_info.locales !== "undefined" && window.banner_info.locales.length > 1) {
-      var locales = window.banner_info.locales;
-      var languages = document.createElement("div");
-
-      var label = document.createElement("span");
-      label.setAttribute("class", "_wb_no-mobile");
-      label.appendChild(document.createTextNode(window.banner_info.choiceLabel + " "));
-      languages.appendChild(label);
-
-      for(var i = 0; i < locales.length; i++) {
-        var locale = locales[i];
-        var langLink = document.createElement("a");
-        langLink.setAttribute("href", "#");
-        langLink.addEventListener("click", this.changeLanguage.bind(this, locale));
-        langLink.appendChild(document.createTextNode(locale));
-
-        languages.appendChild(langLink);
-        if (i !== locales.length - 1) {
-            languages.appendChild(document.createTextNode(" / "));
-        }
-      }
-
-      ancillaryLinks.appendChild(languages);
-    }
-
-    this.banner.appendChild(ancillaryLinks);
-
-    document.body.insertBefore(this.banner, document.body.firstChild);
-  };
-
-  /**
-   * @desc Converts a timestamp to a date string. If is_gmt is truthy then
-   * the returned data string will be the results of date.toGMTString otherwise
-   * its date.toLocaleString()
-   * @param {?string} ts - The timestamp to receive the correct date string for
-   * @param {boolean} is_gmt - Is the returned date string to be in GMT time
-   * @returns {string}
-   */
-  DefaultBanner.prototype.ts_to_date = function(ts, is_gmt) {
-    if (!ts) {
-      return '';
-    }
-
-    if (ts.length < 14) {
-      ts += '00000000000000'.substr(ts.length);
-    }
-
-    var datestr =
-      ts.substring(0, 4) +
-      '-' +
-      ts.substring(4, 6) +
-      '-' +
-      ts.substring(6, 8) +
-      'T' +
-      ts.substring(8, 10) +
-      ':' +
-      ts.substring(10, 12) +
-      ':' +
-      ts.substring(12, 14) +
-      '-00:00';
-
-    var date = new Date(datestr);
-
-    if (is_gmt) {
-      return date.toGMTString();
-    } else {
-      return date.toLocaleString(window.banner_info.locale);
-    }
-  };
-
-  /**
-   * @desc Updates the contents displayed by the banner
-   * @param {?string} url - The URL of the replayed page to be displayed in the banner
-   * @param {?string} ts - A timestamp to be displayed in the banner
-   * @param {boolean} is_live - Are we in live mode
-   * @param {?string} title - The title of the replayed page to be displayed in the banner
-   */
-  DefaultBanner.prototype.set_banner = function(url, ts, is_live, title) {
-    var capture_str;
-    var title_str;
-
-    if (!url) {
-      this.captureInfo.innerHTML = window.banner_info.loadingLabel;
-      this.bannerUrlSet = false;
-      return;
-    }
-
-    if (!ts) {
-      return;
-    }
-
-    if (title) {
-      capture_str = title;
-    } else {
-      capture_str = url;
-    }
-
-    title_str = capture_str;
-
-    capture_str = "<b id='title_or_url' title='" + capture_str + "'>" + capture_str + "</b>";
-
-    capture_str += "<span class='_wb_capture_date'>";
-
-    if (is_live) {
-      title_str = window.banner_info.liveMsg + " " + title_str;
-      capture_str += "<b>" + window.banner_info.liveMsg + "&nbsp;</b>";
-    }
-
-    capture_str += this.ts_to_date(ts, window.banner_info.is_gmt);
-    capture_str += "</span>";
-
-    this.calendarLink.setAttribute("href", window.banner_info.prefix + "*/" + url);
-    this.calendarLink.style.display = is_live ? "none" : "";
-
-    this.captureInfo.innerHTML = capture_str;
-
-    window.document.title = title_str;
-
-    this.bannerUrlSet = true;
-  };
-
-  // all banners will expose themselves by adding themselves as WBBanner on window
-  window.WBBanner = new DefaultBanner();
-
-  // if wbinfo.url is set and not-framed, init banner in content frame
-  if (window.wbinfo && window.wbinfo.url && !window.wbinfo.is_framed) {
-    if (document.readyState === "loading") {
-      document.addEventListener("DOMContentLoaded", function() {
-        window.WBBanner.init();
-      });
-    } else {
-      window.WBBanner.init();
-    }
-  }
-
-})();

pywb/static/loading-spinner/loading-spinner.js

@@ -0,0 +1,160 @@
+const smallSize = "75px";
+
+
+class LoadingSpinner {
+  static #instanceCount = 0;
+
+  constructor(config={}) {
+    this.config = {initialState:true, animationDuration:500, text:'Loading...', ...config};
+
+    if (LoadingSpinner.#instanceCount > 0) {
+      throw new Error('Cannot make a second loading spinner (aka progress indicator)');
+    }
+    LoadingSpinner.#instanceCount++;
+
+
+    const uuid = Math.floor(Math.random()*1000);
+    this.classes = {
+      el: `loading-spinner-${uuid}`,
+      mask: `loading-spinner-mask-${uuid}`,
+      hidden: `hidden-${uuid}`,
+      spinning: `spinning-${uuid}`
+    };
+
+    this.state = config.initialState;
+    this.addStyles();
+    this.addDom();
+  }
+
+  toggle() {
+    if (this.state) {
+      this.setOn();
+    } else {
+      this.setOff();
+    }
+  }
+
+  setOn() {
+    this.state = true;
+    this.el.classList.remove(this.classes.hidden);
+    setTimeout(function setSpinning() {
+      this.el.classList.add(this.classes.spinning);
+    }.bind(this), 10);
+  }
+
+  setOff() {
+    this.state = false;
+    this.el.classList.remove(this.classes.spinning);
+    setTimeout(function setHidden() {
+      this.el.classList.add(this.classes.hidden);
+    }.bind(this), this.config.animationDuration);
+  }
+
+  addDom() {
+    const text = this.config.text;
+    const dom = `
+<div class="${this.classes.mask} ${this.classes[this.config.initialState ? 'spinning':'hidden']}">
+  <div class="${this.classes.el}">
+    <div data-loading-spinner="circle1"></div>
+    <div data-loading-spinner="circle2"></div>
+    <div data-loading-spinner="circle3"></div>
+    <div data-loading-spinner="circle4"></div>
+    <span data-loading-spinner="text">${text}</span>
+  </div>
+</div>`;
+    const wrapEl = document.createElement('div');
+    wrapEl.innerHTML = dom;
+    this.el = wrapEl.firstElementChild;
+    document.getElementsByTagName('body')[0].appendChild(this.el);
+  }
+
+  addStyles() {
+    const duration = this.config.animationDuration;
+    const stylesheetEl = document.createElement('style');
+    document.head.appendChild(stylesheetEl);
+
+    const rules = [`
+.${this.classes.mask} {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100vw;
+    height: ${this.config.isSmall ? smallSize : "100vh"};
+    z-index: 900;
+
+    display: flex;
+    justify-content: center;
+    align-items: center;
+
+    background-color: rgba(255,255,255, .85);
+
+    opacity: 0;
+    transition: opacity ${duration}ms ease-in;
+}`,`
+.${this.classes.mask}.${this.classes.spinning} {
+    opacity: 1;
+}`,`
+.${this.classes.mask}.${this.classes.hidden} {
+    display: none;
+}`,`
+.${this.classes.el} {
+    position: relative;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    width: ${this.config.isSmall ? smallSize : "200px"};
+    height: ${this.config.isSmall ? smallSize : "200px"};
+}`,`
+[data-loading-spinner^=circle] {
+    position: absolute;
+    margin: 0;
+    border-radius: 50%;
+    border-left-color: transparent;
+    border-right-color: transparent;
+}`,`
+[data-loading-spinner=circle1] {
+    border: 3px solid #444444;/* #0D4B9F; */
+    width: 70%;
+    height: 70%;
+    animation: rotate 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}`,`
+[data-loading-spinner=circle2] {
+    border: 3px solid #ddd;/* #E0EDFF; */
+    width: 80%;
+    height: 80%;
+    animation: rotateReverse 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}`,`
+[data-loading-spinner=circle3] {
+    border: 3px solid #656565;/* #005CDC; */
+    width: 90%;
+    height: 90%;
+    animation: rotate 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}`,`
+[data-loading-spinner=circle4] {
+    border: 3px solid #aaa; /* #94B6E5; */
+    width: 100%;
+    height: 100%;
+    animation: rotateReverse 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}`,`
+@keyframes rotate {
+    from {
+        transform: rotateZ(-360deg)
+    }
+    to {
+        transform: rotateZ(0deg)
+    }
+}`,`
+@keyframes rotateReverse {
+    from {
+        transform: rotateZ(360deg)
+    }
+    to {
+        transform: rotateZ(0deg)
+    }
+}`,`
+[data-loading-spinner=text] {
+    font-size: 15px;
+}`];
+      rules.forEach(rule => stylesheetEl.sheet.insertRule(rule));
+  }
+}

pywb/static/loading-spinner/test.html

@@ -0,0 +1,12 @@
+<html>
+<head>
+<script src="loading-spinner.js"></script>
+</head>
+<body>
+<button onclick="loadingSpinner.setOn()">load...</button>
+<script>
+    const loadingSpinner = new LoadingSpinner();
+    loadingSpinner.el.addEventListener('click', e => loadingSpinner.setOff());
+</script>
+</body>
+</html>

pywb/static/query.js

@@ -57,14 +57,6 @@ function RenderCalendar(init) {
   };
   // regex for extracting the filter constraints and filter mods to human explanation
   this.filterRE = /filter([^a-z]+)([a-z]+):(.+)/i;
-  this.filterMods = {
-    '=': 'Contains',
-    '==': 'Matches Exactly',
-    '=~': 'Matches Regex',
-    '=!': 'Does Not Contains',
-    '=!=': 'Is Not',
-    '=!~': 'Does Not Begins With'
-  };
   this.text = init.text;
   this.versionString = null;
 }
@@ -371,16 +363,13 @@ RenderCalendar.prototype.createContainers = function() {
         },
         { tag: 'textNode', value: ' ' },
         {
-          tag: 'b',
-          child: {
-            tag: 'textNode',
-            value: '',
-            ref: function(refToElem) {
-              renderCal.containers.versionsTextNode = refToElem;
-            }
+          tag: 'textNode',
+          value: '',
+          ref: function(refToElem) {
+            renderCal.containers.versionsTextNode = refToElem;
           }
         },
-        { tag: 'textNode', value: ' of ' + this.queryInfo.url }
+        { tag: 'b', innerText: ' ' + this.queryInfo.url }
       ]
     });
     // create the row that will hold the results of the regular query
@@ -436,15 +425,14 @@ RenderCalendar.prototype.createContainers = function() {
     return;
   }
   // create the advanced results query info DOM structure
-  var forString = ' for ';
   var forElems;
 
   if (this.queryInfo.searchParams.matchType) {
-    forString = ' for matching ';
+    forString = ' ' + this.text.matching + ' ';
     forElems = [
       { tag: 'b', innerText: this.queryInfo.url },
-      { tag: 'textNode', value: ' by ' },
-      { tag: 'b', innerText: this.queryInfo.searchParams.matchType }
+      { tag: 'textNode', value: ' ' + this.text.by + ' ' },
+      { tag: 'b', innerText: this.text.types[this.queryInfo.searchParams.matchType] }
     ];
   } else {
     forElems = [{ tag: 'b', innerText: this.queryInfo.url }];
@@ -463,23 +451,21 @@ RenderCalendar.prototype.createContainers = function() {
         },
         {
           tag: 'b',
-          children: [
-            {
-              tag: 'textNode',
-              value: '',
-              ref: function(refToElem) {
-                renderCal.containers.countTextNode = refToElem;
-              }
-            },
-            { tag: 'textNode', value: ' ' },
-            {
-              tag: 'textNode',
-              value: '',
-              ref: function(refToElem) {
-                renderCal.containers.versionsTextNode = refToElem;
-              }
+          child: {
+            tag: 'textNode',
+            value: '',
+            ref: function(refToElem) {
+              renderCal.containers.countTextNode = refToElem;
             }
-          ]
+          }
+        },
+        { tag: 'textNode', value: ' ' },
+        {
+          tag: 'textNode',
+          value: '',
+          ref: function(refToElem) {
+            renderCal.containers.versionsTextNode = refToElem;
+          }
         },
         { tag: 'textNode', value: forString }
       ].concat(forElems)
@@ -508,7 +494,7 @@ RenderCalendar.prototype.createContainers = function() {
         {
           tag: 'p',
           className: 'text-center mb-0 mt-1',
-          innerText: 'Filtering by'
+          innerText: filteringBy
         },
         {
           tag: 'ul',
@@ -614,13 +600,13 @@ RenderCalendar.prototype.renderAdvancedSearchPart = function(cdxObj) {
   if (cdxObj.mime) {
     displayedInfo.push({
       tag: 'small',
-      innerText: 'Mime Type: ' + cdxObj.mime
+      innerText: this.text.mimeType + cdxObj.mime
     });
   }
   if (cdxObj.status) {
     displayedInfo.push({
       tag: 'small',
-      innerText: 'HTTP Status: ' + cdxObj.status
+      innerText: this.text.httpStatus + cdxObj.status
     });
   }
   displayedInfo.push({
@@ -785,6 +771,11 @@ RenderCalendar.prototype.addRegYearMonthDayListItem = function(
        a[href="replay url"]
        span[id=count_ts].badge.badge-info.badge-pill.float-right
    */
+    const options = {
+      dateStyle: 'long',
+      timeStyle: 'medium',
+    };
+    var dateTimeString = this.tsToDate(cdxObj.timestamp, false, options);
     this.createAndAddElementTo(ymlDL, {
       tag: 'li',
       className: 'list-group-item',
@@ -795,17 +786,7 @@ RenderCalendar.prototype.addRegYearMonthDayListItem = function(
             href: this.prefix + cdxObj.timestamp + '/' + cdxObj.url,
             target: '_blank'
           },
-          innerText:
-            timeInfo.month +
-            ' ' +
-            timeInfo.day +
-            this.dateOrdinal(timeInfo.day) +
-            ', ' +
-            timeInfo.year +
-            ' ' +
-            ' at ' +
-            timeInfo.time +
-            '  '
+          innerText: dateTimeString
         },
         {
           tag: 'span',
@@ -960,7 +941,7 @@ RenderCalendar.prototype.niceFilterDisplay = function() {
       filterList.push({
         tag: 'li',
         className: 'list-group-item',
-        innerText: match[2] + ' ' + this.filterMods[match[1]] + ' ' + match[3]
+        innerText: match[2] + ' ' + filterMods[match[1]] + ' "' + match[3] + '"'
       });
     }
   }
@@ -975,11 +956,11 @@ RenderCalendar.prototype.niceDateRange = function() {
   var from = this.queryInfo.searchParams.from;
   var to = this.queryInfo.searchParams.to;
   if (from && to) {
-    return 'From ' + from + ' to ' + to;
+    return [text.from, from, text.until, to].join(' ');
   } else if (from) {
-    return 'From ' + from + ' until ' + 'present';
+    return [text.from, from, text.until, text.present].join(' ');
   }
-  return 'From earliest until ' + to;
+  return [text.from, text.earliest, text.until, to].join(' ');
 };
 
 /**
@@ -1020,32 +1001,14 @@ RenderCalendar.prototype.displayYearMonthDaysListId = function(year, month) {
   return '_' + year + '-' + month + '-Display-Days-List';
 };
 
-/**
- * Returns a numbers ordinal string
- * @param {number} d - The number to receive the ordinal string for
- * @returns {string}
- */
-RenderCalendar.prototype.dateOrdinal = function(d) {
-  if (d > 3 && d < 21) return 'th';
-  switch (d % 10) {
-    case 1:
-      return 'st';
-    case 2:
-      return 'nd';
-    case 3:
-      return 'rd';
-    default:
-      return 'th';
-  }
-};
-
 /**
  * Converts the supplied timestamp to either a local data string or a gmt string (if is_gmt is true)
  * @param {string} ts - The timestamp to be converted to a string
  * @param {boolean} [is_gmt] - Should the timestamp be converted to a gmt string
+ * @param {Object} [options] - String formatting options
  * @returns {string}
  */
-RenderCalendar.prototype.tsToDate = function(ts, is_gmt) {
+RenderCalendar.prototype.tsToDate = function(ts, is_gmt, options) {
   if (ts.length < 14) return ts;
   var datestr =
     ts.substring(0, 4) +
@@ -1062,7 +1025,7 @@ RenderCalendar.prototype.tsToDate = function(ts, is_gmt) {
     '-00:00';
 
   var date = new Date(datestr);
-  return is_gmt ? date.toGMTString() : date.toLocaleString();
+  return is_gmt ? date.toUTCString() : date.toLocaleString(document.documentElement.lang, options);
 };
 
 /**

pywb/static/search.js

@@ -1,14 +1,6 @@
 var dtRE = /^\d{4,14}$/;
 var didSetWasValidated = false;
 var showBadDateTimeClass = 'show-optional-bad-input';
-var filterMods = {
-  '=': 'Contains',
-  '==': 'Matches Exactly',
-  '=~': 'Matches Regex',
-  '=!': 'Does Not Contains',
-  '=!=': 'Is Not',
-  '=!~': 'Does Not Begins With'
-};
 
 var elemIds = {
   filtering: {
@@ -22,16 +14,34 @@ var elemIds = {
   },
   dateTime: {
     from: 'dt-from',
+    fromTime: 'ts-from',
     fromBad: 'dt-from-bad',
     to: 'dt-to',
+    toTime: 'ts-to',
     toBad: 'dt-to-bad'
   },
   match: 'match-type-select',
   url: 'search-url',
   form: 'search-form',
-  resultsNewWindow: 'open-results-new-window'
+  resultsNewWindow: 'open-results-new-window',
+  advancedOptions: 'advanced-options',
+  resetSearchForm: 'reset-search-form',
 };
 
+function resetSearchForm(event) {
+  for (const field of [
+    elemIds.url,
+    elemIds.match,
+    elemIds.dateTime.from,
+    elemIds.dateTime.fromTime,
+    elemIds.dateTime.to,
+    elemIds.dateTime.toTime,
+  ]) {
+    document.getElementById(field).value = '';
+  }
+  clearFilters(event);
+}
+
 function makeCheckDateRangeChecker(dtInputId, dtBadNotice) {
   var dtInput = document.getElementById(dtInputId);
   dtInput.onblur = function() {
@@ -64,7 +74,7 @@ function makeCheckDateRangeChecker(dtInputId, dtBadNotice) {
 
 function createAndAddNoFilter(filterList) {
   var nothing = document.createElement('li');
-  nothing.innerText = 'No Filter';
+  nothing.innerText = noFilter;
   nothing.id = elemIds.filtering.nothing;
   filterList.appendChild(nothing);
 }
@@ -77,19 +87,24 @@ function addFilter(event) {
   if (!expr) return;
   var filterExpr = 'filter' + modifier + by + ':' + expr;
   var filterList = document.getElementById(elemIds.filtering.list);
+  var previousFilters = filterList.children;
+  for (var i = 0; i < previousFilters.length; ++i) {
+    var filterData = previousFilters[i].dataset;
+    if (filterData && filterData.filter && filterData.filter == filterExpr) return;
+  }
   var filterNothing = document.getElementById(elemIds.filtering.nothing);
   if (filterNothing) {
     filterList.removeChild(filterNothing);
   }
   var li = document.createElement('li');
   li.innerText =
-    'By ' +
     by[0].toUpperCase() +
     by.substr(1) +
     ' ' +
     filterMods[modifier] +
-    ' ' +
-    expr;
+    ' "' +
+    expr +
+    '"';
   li.dataset.filter = filterExpr;
   var nukeButton = document.createElement('button');
   nukeButton.type = 'button';
@@ -109,6 +124,7 @@ function addFilter(event) {
   };
   li.appendChild(nukeButton);
   filterList.appendChild(li);
+  return true;
 }
 
 function clearFilters(event) {
@@ -139,11 +155,13 @@ function performQuery(url) {
   }
   var fromT = document.getElementById(elemIds.dateTime.from).value;
   if (fromT) {
-    query.push('from=' + fromT.trim());
+    fromT += document.getElementById(elemIds.dateTime.fromTime).value;
+    query.push('from=' + fromT.replace(/[^0-9]/g, ''));
   }
   var toT = document.getElementById(elemIds.dateTime.to).value;
   if (toT) {
-    query.push('to=' + toT.trim());
+    toT += document.getElementById(elemIds.dateTime.toTime).value;
+    query.push('to=' + toT.replace(/[^0-9]/g, ''));
   }
   var builtQuery = query.join('&');
   if (document.getElementById(elemIds.resultsNewWindow).checked) {
@@ -158,6 +176,24 @@ function performQuery(url) {
   }
 }
 
+function validateFields(form) {
+  if (!didSetWasValidated) {
+    form.classList.add('was-validated');
+    didSetWasValidated = true;
+  }
+}
+
+function submitForm(event, form, searchURLInput) {
+  event.preventDefault();
+  event.stopPropagation();
+  var url = searchURLInput.value;
+  if (!url) {
+    validateFields(form);
+    return;
+  }
+  performQuery(url);
+}
+
 $(document).ready(function() {
   $('[data-toggle="tooltip"]').tooltip({
     container: 'body',
@@ -171,21 +207,21 @@ $(document).ready(function() {
     elemIds.dateTime.to,
     document.getElementById(elemIds.dateTime.toBad)
   );
+  document.getElementById(elemIds.resetSearchForm).onclick = resetSearchForm;
   document.getElementById(elemIds.filtering.add).onclick = addFilter;
   document.getElementById(elemIds.filtering.clear).onclick = clearFilters;
   var searchURLInput = document.getElementById(elemIds.url);
   var form = document.getElementById(elemIds.form);
   form.addEventListener('submit', function(event) {
-    event.preventDefault();
-    event.stopPropagation();
-    var url = searchURLInput.value;
-    if (!url) {
-      if (!didSetWasValidated) {
-        form.classList.add('was-validated');
-        didSetWasValidated = true;
+    submitForm(event, form, searchURLInput);
+  });
+  var filteringExpression = document.getElementById(elemIds.filtering.expression);
+  filteringExpression.addEventListener("keypress", function(event) {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      if (! addFilter()) {
+        submitForm(event, form, searchURLInput);
       }
-      return;
     }
-    performQuery(url);
   });
 });

pywb/static/vue_banner.css

@@ -0,0 +1,4 @@
+#wb_iframe_div, #replay_iframe {
+  width: 100%;
+  height: 100%;
+}

pywb/static/wb_frame.js

@@ -65,6 +65,8 @@ ContentFrame.prototype.init_iframe = function() {
     return;
   }
 
+  this.iframe.setAttribute("name", "___wb_replay_top_frame");
+
   this.extract_prefix();
   if (window.WBBanner) {
     this.wbBanner = window.WBBanner;
@@ -229,7 +231,13 @@ ContentFrame.prototype.initBannerUpdateCheck = function(newUrl, newTs) {
  * operating in live mode
  */
 ContentFrame.prototype.load_url = function(newUrl, newTs) {
-  this.iframe.src = this.make_url(newUrl, newTs, true);
+  var newUrl = this.make_url(newUrl, newTs, true);
+  if (this.iframe.src === newUrl) {
+    return;
+  }
+
+  this.iframe.src = newUrl;
+
   if (this.wbBanner) {
     this.initBannerUpdateCheck(newUrl, newTs);
   }

pywb/static/wombatProxyMode.js

@@ -1,6 +1,6 @@
 /*
 Wombat.js client-side rewriting engine for web archive replay
-Copyright (C) 2014-2020 Webrecorder Software, Rhizome, and Contributors. Released under the GNU Affero General Public License.
+Copyright (C) 2014-2024 Webrecorder Software, Rhizome, and Contributors. Released under the GNU Affero General Public License.
 
 This file is part of wombat.js, see https://github.com/webrecorder/wombat.js for the full source
 Wombat.js is part of the Webrecorder project (https://github.com/webrecorder)

pywb/static/wombatWorkers.js

@@ -1,6 +1,6 @@
 /*
 Wombat.js client-side rewriting engine for web archive replay
-Copyright (C) 2014-2020 Webrecorder Software, Rhizome, and Contributors. Released under the GNU Affero General Public License.
+Copyright (C) 2014-2024 Webrecorder Software, Rhizome, and Contributors. Released under the GNU Affero General Public License.
 
 This file is part of wombat.js, see https://github.com/webrecorder/wombat.js for the full source
 Wombat.js is part of the Webrecorder project (https://github.com/webrecorder)

pywb/templates/banner.html

@@ -17,13 +17,16 @@ window.banner_info = {
     locales: {{ locales }},
     locale_prefixes: {{ get_locale_prefixes() | tojson }},
     prefix: "{{ wb_prefix }}",
-    staticPrefix: "{{ static_prefix }}"
+    staticPrefix: "{{ static_prefix }}",
+
+    logoImg: "{{ ui.logo }}"
 };
 </script>
+<script src="{{ static_prefix }}/loading-spinner/loading-spinner.js"></script>
+<script src="{{ static_prefix }}/vue/vueui.js"></script>
+<link rel="stylesheet" href='{{ static_prefix }}/vue_banner.css'/>
 
-<!-- default banner, create through js -->
-<script src='{{ static_prefix }}/default_banner.js'> </script>
-<link rel='stylesheet' href='{{ static_prefix }}/default_banner.css'/>
+{% include 'bootstrap_jquery.html' ignore missing %}
 
 {% endautoescape %}
 {% endif %}

pywb/templates/base.html

@@ -6,13 +6,7 @@
 
         <title>{% block title %}{% endblock %}</title>
 
-        <!-- jquery and bootstrap dependencies query view -->
-        <link rel="stylesheet" href="{{ static_prefix }}/css/bootstrap.min.css"/>
-        <link rel="stylesheet" href="{{ static_prefix }}/css/font-awesome.min.css">
-        <link rel="stylesheet" href="{{ static_prefix }}/css/base.css">
-
-        <script src="{{ static_prefix }}/js/jquery-latest.min.js"></script>
-        <script src="{{ static_prefix }}/js/bootstrap.min.js"></script>
+        {% include 'bootstrap_jquery.html' ignore missing %}
 
         {% block head %}
         {% include 'head.html' ignore missing %}

pywb/templates/bootstrap_jquery.html

@@ -0,0 +1,6 @@
+<link rel="stylesheet" href="{{ static_prefix }}/css/bootstrap.min.css"/>
+<link rel="stylesheet" href="{{ static_prefix }}/css/font-awesome.min.css">
+<link rel="stylesheet" href="{{ static_prefix }}/css/base.css">
+
+<script src="{{ static_prefix }}/js/jquery-latest.min.js"></script>
+<script src="{{ static_prefix }}/js/bootstrap.min.js"></script>

pywb/templates/custom_banner.html

@@ -0,0 +1 @@
+<!-- Add custom banner here. Used only in non-framed replay. -->

pywb/templates/error.html

@@ -1,9 +1,9 @@
 {% extends "base.html" %}
 {% block title %}{{ _('Pywb Error') }}{% endblock %}
 {% block body %}
-<div class="container text-danger">
+<div class="container text-danger error">
     <div class="row justify-content-center">
-        <h2 class="display-2">Pywb Error</h2>
+        <h2 class="display-2">{{ _('Pywb Error') }}</h2>
     </div>
     <div class="row">
         <div class="col-12 text-center">

pywb/templates/footer.html

@@ -0,0 +1,2 @@
+{# place content to be added at the very end of the <body> tag in this file below #}
+

pywb/templates/frame_insert.html

@@ -18,9 +18,30 @@ html, body
 
 {{ banner_html }}
 
+{% include 'vue_loc.html' %}
+
 </head>
 <body style="margin: 0px; padding: 0px;">
 
+<div id="app" style="width: 100%; height: 200px"></div>
+<script>
+  VueUI.main({
+      staticPrefix: "{{ static_prefix }}",
+      url: "{{ url }}",
+      prefix: "{{ wb_prefix }}",
+      timestamp: "{{ timestamp }}",
+      logoUrl: "{{ ui.logo }}",
+      navbarBackground: "{{ ui.navbar_background_hex | default('f8f9fa') }}",
+      navbarColor: "{{ ui.navbar_color_hex | default('212529') }}",
+      navbarLightButtons: "{{ ui.navbar_light_buttons }}",
+      logoHomeUrl: "{{ ui.logo_home_url }}",
+      disablePrinting: "{{ ui.disable_printing }}",
+      allLocales: allLocales
+    },
+    "{{ env.pywb_lang | default('en') }}",
+    i18nStrings);
+</script>
+
 <div id="wb_iframe_div">
 <iframe id="replay_iframe" frameborder="0" seamless="seamless" scrolling="yes" class="wb_iframe" allow="autoplay; fullscreen"></iframe>
 </div>

pywb/templates/head.html

@@ -0,0 +1 @@
+{# place optional content to be injected into the <head> of every page in this file below #}

pywb/templates/head_insert.html

@@ -27,6 +27,7 @@
   wbinfo.proxy_magic = "{{ env.pywb_proxy_magic }}";
   wbinfo.static_prefix = "{{ static_prefix }}/";
   wbinfo.enable_auto_fetch = {{ config.enable_auto_fetch | tobool }};
+  wbinfo.target_frame = "___wb_replay_top_frame";
 </script>
 {% if env.pywb_proxy_magic %}
 {% set whichWombat = 'wombatProxyMode.js' %}
@@ -61,7 +62,11 @@
 
 {% endif %}
 
-{{ banner_html }}
+{% if not is_framed %}
+
+{{ custom_banner_html }}
+
+{% endif %}
 
 {% endautoescape %}
 

pywb/templates/header.html

@@ -1,5 +1,6 @@
+{# place content to be added at the very beginning of the <body> tag in this file below #}
 <header>
-   {% if not err_msg and locales|length > 1 %}
+   {% if not err_msg and locales|length > 1 and (not ui or not ui.vue_calendar_ui) %}
     <div class="language-select">
         {{ _('Language:') }}
         <ul role="listbox" aria-activedescendant="{{ env.pywb_lang | default(default_locale) }}" aria-labelledby="{{ _('Language select') }}">

pywb/templates/index.html

@@ -10,7 +10,7 @@
             {% for route in routes %}
                 <li>
                     <a href="{{ env['pywb.app_prefix'] + ('/' + env.pywb_lang if env.pywb_lang else '') + '/' + route }}">{{ '/' + route }}</a>
-                    {% if all_metadata and all_metadata[route] %}
+                    {% if all_metadata and all_metadata[route] and all_metadata[route].title %}
                         ({{ all_metadata[route].title }})
                     {% endif %}
                 </li>

pywb/templates/instructions.html

@@ -0,0 +1,216 @@
+<div class="modal fade" id="searchInstructions" tabindex="-1" role="dialog" aria-labelledby="searchInstructionsTitle" aria-hidden="true">
+    <div class="modal-dialog modal-lg" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h6 class="modal-title text-muted" id="searchInstructionsTitle">{{ _("Search instructions") }}</h6>
+                <button type="button" class="close" data-dismiss="modal" aria-label="{{ _('Close') }}">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <div class="modal-body">
+                <h5>{{ _("URL") }}</h5>
+                <table class="table table-hover table-condensed">
+                    <tr>
+                        <td>
+                            <p>
+                                {%trans%}A URL consists of several parts:{%endtrans%}
+                                {%trans%}<code>protocol</code>://<code>host</code>:<code>port</code>/<code>path</code>?<code>query</code>{%endtrans%}
+                            </p>
+
+                            <p>
+                                {%trans%}The <code>protocol://</code> prefix is ignored when searching as it's not part of the searchable data.{%endtrans%}
+                            </p>
+                            <p>
+                                {%trans%}A leading <kbd>www.</kbd> in the <code>host</code> will also be ignored for the same reason.{%endtrans%}
+                            </p>
+
+                            <p>
+                                {%trans%}The <code>host</code> contains one or more parts separated by periods (<kbd>.</kbd>).{%endtrans%}
+                                {%trans%}The part before the first period is called the <code>hostname</code>.{%endtrans%}
+                                {%trans%}The part after the last period is the <code>top level domain</code>.{%endtrans%}
+                                {%trans%}Every part added to the left of the top level domain <code>sub-domain</code>.{%endtrans%}
+                                {%trans%}I.e. <code>x.y.z</code> is a <code>sub-domain</code> of <code>y.z</code>{%endtrans%}
+                                {%trans%}which in turn is a <code>sub-domain</code> of the <code>top level domain</code> <code>z</code>{%endtrans%}
+                            </p>
+
+                            <p>
+                                {%trans%}See <em>Match Type</em> below for interpretations of the search string.{%endtrans%}
+                            </p>
+                        </td>
+                    </tr>
+                </table>
+
+                <h5>{{ _("Results Display") }}</h5>
+                <table class="table table-hover table-condensed">
+                    <tr>
+                        <td>
+                            <p>
+                                {%trans%}For the <em>Default</em> search mode, the results are shown in a calendar view unless a filter is also added.{%endtrans%}
+                                {%trans%}For all other cases the results will be displayed in a list.{%endtrans%}
+                            </p>
+                        </td>
+                    </tr>
+                </table>
+
+                <h5>{{ _("Search Options") }}</h5>
+                <h6>{{ _("Match Type") }}</h6>
+                <p> {{ _("There are four different search modes:") }}</p>
+
+                <table class="table table-hover table-condensed">
+                    <tr>
+                        <td><em>{{ _("Default") }}</em></td>
+                        <td>
+                            <p>
+                                {%trans%}In the default mode the exact URL (minus the ignored prefixes mentioned above) is searched for.{%endtrans%}
+                                {%trans%}If one leading or trailing wildcard asterisk (<kbd>*</kbd>) is added, see <em>Prefix</em> and <em>Domain</em> below.{%endtrans%}
+                            </p>
+                            <p class="text-muted">
+                                {%trans%}Any other asterisks will be considered literal parts of the search string.{%endtrans%}
+                                {%trans%}Hence, adding both a leading and a trailing wildcard asterisk is not possible.{%endtrans%}
+                            </p>
+
+                            {%trans%}Example:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/206</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Default") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/206&match-type-select=');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/206&match-type-select=', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+
+                    <tr>
+                        <td><em>{{ _("Prefix") }}</em></td>
+                        <td>
+                            <p>
+                                {%trans%}This will return all URL:s that begin with the given string.{%endtrans%}
+                                {%trans%}It returns the same results as <em>Default</em> with a trailing wildcard asterisk.{%endtrans%}
+                            </p>
+
+                            {%trans%}Examples:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/2</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Prefix") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/2*</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Default") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/2*&match-type-select=');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/2*&match-type-select=', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+
+                    <tr>
+                        <td><em>{{ _("Host") }}</em></td>
+                        <td>
+                            <p>
+                                {%trans%}This will ignore any path and query parts of the URL and return all URL:s with the specified <code>host</code> part.{%endtrans%}
+                            </p>
+
+                            {%trans%}Example:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Host") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/&match-type-select=host');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/&match-type-select=host', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+
+                    <tr>
+                        <td><em>{{ _("Domain") }}</em></td>
+                        <td>
+                            <p>
+                                {%trans%}This is similar to the previous but doesn't require the whole <code>host</code>.{%endtrans%}
+                                {%trans%}It returns the same results as <em>Default</em> with a leading wildcard asterisk and a period (i.e. <kbd>*.</kbd>).{%endtrans%}
+                                {%trans%}The leading wildcard matches zero or more <code>sub-domains</code> as well as zero or one <code>hostname</code>.{%endtrans%}
+                            </p>
+
+                            {%trans%}Examples:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>cat/</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Domain") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=cat/&match-type-select=domain');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=cat/&match-type-select=domain', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>*.cat/</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Default") }}</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=*.cat/&match-type-select=');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=*.cat/&match-type-select=', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+                </table>
+
+                <h6>{{ _("Date/Time Range") }}</h6>
+                <table class="table table-hover table-condensed">
+                    <tr>
+                        <td>
+                            <p>
+                                {%trans%}One may specify a start and/or an end timestamp to further restrict the search - both are inclusive.{%endtrans%}
+                                {%trans%}The timestamps consist of a date and an optional time of day.{%endtrans%}
+                                {%trans%}The layout of these input fields are subject to which browser is used.{%endtrans%}
+                            </p>
+
+                            {%trans%}Example:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/2</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Prefix") }}</strong></em> &amp; <em>{{ _("From") }}: <strong>2022-02-02 09:00</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix&dt-from=2022-02-02&ts-from=09:00');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix&dt-from=2022-02-02&ts-from=09:00', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+                </table>
+
+                <h6>{{ _("Filtering") }}</h6>
+                <table class="table table-hover table-condensed">
+                    <tr>
+                        <td>
+                            <p>
+                                {%trans%}Finally one may add extra filters for Mime Type, Status and URL.{%endtrans%}
+                                {%trans%}For each filter one needs to specify one of the three attributes, one of a set of relations and a string.{%endtrans%}
+                                {%trans%}If more than one filter is added, they will all be applied to the list of results.{%endtrans%}
+                            </p>
+                            <p class="text-muted">{%trans%}Remember to actually add the filter before submitting the search.{%endtrans%}</p>
+
+                            {%trans%}Example:{%endtrans%}
+                            <p class="ml-5 text-lowercase">
+                                <em>{{ _("URL") }}: <strong>https://http.cat/2/</strong></em> &amp; <em>{{ _("Match Type") }}: <strong>{{ _("Prefix") }}</strong></em> &amp; <em>{{ _("Filtering") }}: <strong>{{ _("HTTP Status") }} {{ _("Is Not") }} "301"</strong></em>
+                                <span class="float-right">
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix&filter-by=status&filter-modifier==!=&filter-expression=301');" class="btn btn-outline-info" role="button" aria-label="{{ _('Fill') }}">{{ _('Fill') }}</button>
+                                    <button onclick="fillForm('search-url=https://http.cat/2&match-type-select=prefix&filter-by=status&filter-modifier==!=&filter-expression=301', true);" class="btn btn-outline-primary" role="button" aria-label="{{ _('Search') }}">{{ _('Search') }}</button>
+                                </span>
+                            </p>
+                        </td>
+                    </tr>
+                </table>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+  function fillForm(query, search = false) {
+    $('#searchInstructions').modal('hide');
+    $('#advancedOptions').collapse('show');
+    for (const item of query.split('&')) {
+      var pair = item.split('=');
+      var field = document.getElementById(pair[0]);
+      if (field) field.value = pair.slice(1).join('=');
+      if (pair[0] == "filter-expression") addFilter(event);
+    }
+    if (search) $('#search-button').click();
+  }
+</script>

pywb/templates/query.html

@@ -6,13 +6,29 @@
 
 {% block head %}
 {{ super() }}
+
+{% if "filter=" in env['QUERY_STRING'] or "matchType=" in env['QUERY_STRING'] %}
+
 <link rel="stylesheet" href="{{ static_prefix }}/css/query.css">
 <script src="{{ static_prefix }}/js/url-polyfill.min.js"></script>
 <script src="{{ static_prefix }}/query.js"></script>
+
+{% else %}
+
+<script src="{{ static_prefix }}/loading-spinner/loading-spinner.js"></script>
+<script src="{{ static_prefix }}/vue/vueui.js"></script>
+
+{% endif %}
+
+{% include 'vue_loc.html' %}
+
 {% endblock %}
 
 
 {% block body %}
+
+{% if "filter=" in env['QUERY_STRING'] or "matchType=" in env['QUERY_STRING'] %}
+
 <div class="container-fluid">
     <div class="row justify-content-center">
         <h4 class="display-4 text-center text-sm-left p-0">{{ _('Search Results') }}</h4>
@@ -38,15 +54,63 @@
        '11': "{{ _('November') }}",
        '12': "{{ _('December') }}",
       },
-      version: "{{ _('capture') }}",
-      versions: "{{ _('captures') }}",
+      version: "{{ _('capture of') }}",
+      versions: "{{ _('captures of') }}",
       result: "{{ _('result') }}",
       results: "{{ _('results') }}",
+      matching: "{{ _('for matching') }}",
+      by: "{{ _('by') }}",
       viewAllCaptures: "{{ _('View All Captures') }}",
       dateTime: "{{ _('Date Time: ') }}",
+      mimeType: "{{ _('Mime Type: ') }}",
+      httpStatus: "{{ _('HTTP Status: ') }}",
+      types: {
+        'prefix': "{{ _('prefix') }}",
+        'host': "{{ _('host') }}",
+        'domain': "{{ _('domain') }}",
+      },
+      from: "{{ _('From') }}",
+      until: "{{ _('until') }}",
+      present: "{{ _('present') }}",
+      earliest: "{{ _('earliest') }}",
     };
 
+  var filterMods = {
+    '=': "{{ _('Contains') }}",
+    '==': "{{ _('Matches Exactly') }}",
+    '=~': "{{ _('Matches Regex') }}",
+    '=!': "{{ _('Does Not Contain') }}",
+    '=!=': "{{ _('Is Not') }}",
+    '=!~': "{{ _('Does Not Begin With') }}"
+  };
+  var filteringBy = "{{ _('Filtering by') }}";
+  var forString = " {{ _('for') }} ";
   var renderCal = new RenderCalendar({ prefix: "{{ prefix }}", staticPrefix: "{{ static_prefix }}", text: text });
   renderCal.init();
 </script>
+
+{% else %}
+
+<div id="app" style="width: 100%; height: 100%"></div>
+
+<script>
+  VueUI.main({
+      staticPrefix: "{{ static_prefix }}",
+      url: "{{ url }}",
+      prefix: "{{ prefix }}",
+      timestamp: undefined,
+      logoUrl: "{{ ui.logo }}",
+      navbarBackground: "{{ ui.navbar_background_hex | default('f8f9fa') }}",
+      navbarColor: "{{ ui.navbar_color_hex | default('212529') }}",
+      navbarLightButtons: "{{ ui.navbar_light_buttons }}",
+      logoHomeUrl: "{{ ui.logo_home_url }}",
+      disablePrinting: "{{ ui.disable_printing }}",
+      allLocales: allLocales
+    },
+    "{{ env.pywb_lang | default('en') }}",
+    i18nStrings);
+</script>
+
+{% endif %}
+
 {% endblock %}

pywb/templates/search.html

@@ -3,8 +3,18 @@
 {% block head %}
 {{ super() }}
 <script>
-// TODO: cleanup
-window.wb_prefix = "{{ wb_prefix }}";
+  var filterMods = {
+    '=': "{{ _('Contains') }}",
+    '==': "{{ _('Matches Exactly') }}",
+    '=~': "{{ _('Matches Regex') }}",
+    '=!': "{{ _('Does Not Contain') }}",
+    '=!=': "{{ _('Is Not') }}",
+    '=!~': "{{ _('Does Not Begin With') }}"
+  };
+  var noFilter = "{{ _('No Filter') }}";
+
+  // TODO: cleanup
+  window.wb_prefix = "{{ wb_prefix }}";
 </script>
 <script src="{{ static_prefix }}/search.js"></script>
 {% endblock %}
@@ -21,15 +31,22 @@ window.wb_prefix = "{{ wb_prefix }}";
     <form class="needs-validation" id="search-form" novalidate>
         <div class="form-row">
             <div class="col-12">
-                <label for="search-url" class="lead" aria-label="Search For Col">
+                <label for="search-url" class="lead" aria-label="{{ _('Search Collection') }}">
                     {% set coll_title = metadata.title if metadata and metadata.title else coll %}
+                    {% autoescape false %}
                     {% trans %}Search the {{ coll_title }} collection by url:{% endtrans %}
+                    {% endautoescape %}
                 </label>
-                <input aria-label="url" aria-required="true" class="form-control form-control-lg" id="search-url"
+                <a tabindex="0" class="btn btn-sm float-right btn-light" role="button" data-toggle="modal" data-target="#searchInstructions">{{ _('Help') }}</a>
+            </div>
+        </div>
+        <div class="form-row">
+            <div class="col-12">
+                <input aria-label="{{ _('URL') }}" aria-required="true" class="form-control form-control-lg" id="search-url"
                        name="search" placeholder="{{ _('Enter a URL to search for') }}"
-                       title="{{ _('Enter a URL to search for') }}" type="search" required/>
+                       title="{{ _('Enter a URL to search for') }}" type="search" required autofocus />
                 <div class="invalid-feedback">
-                    {% trans %}'Please enter a URL{% endtrans %}
+                    {% trans %}Please enter a URL{% endtrans %}
                 </div>
             </div>
         </div>
@@ -41,23 +58,26 @@ window.wb_prefix = "{{ wb_prefix }}";
                 </div>
             </div>
             <div class="col-7">
-                <button type="submit" class="btn btn-outline-primary float-right" role="button" aria-label="Search">
+                <button type="submit" id="search-button" class="btn btn-primary float-right" role="button" aria-label="{{ _('Search') }}">
                     {% trans %}Search{% endtrans %}
                 </button>
-                <button class="btn btn-outline-info float-right mr-3" type="button" role="button"
-                        data-toggle="collapse" data-target="#advancedOptions"
-                        aria-expanded="false" aria-controls="advancedOptions" aria-label="Advanced Search Options">
-                    {{ _('Advanced Search Options') }}
+                <button class="btn btn-outline-secondary float-right mr-3" type="button" role="button"
+                        data-toggle="collapse" data-target="#advancedOptions" id="advanced-options"
+                        aria-expanded="false" aria-controls="advancedOptions" aria-label="{{ _('Search Options') }}">
+                    {{ _('Search Options') }}
+                </button>
+                <button id="reset-search-form" class="btn btn-outline-danger float-right mr-3" type="button" role="button" aria-label="{{ _('Reset Options') }}">
+                    {{ _('Reset') }}
                 </button>
             </div>
         </div>
         <div class="collapse mt-3" id="advancedOptions">
             <div class="form-group form-row">
-                <label for="match-type-select" class="col-sm-2 col-form-label" aria-label="Match Type">
+                <label for="match-type-select" class="col-sm-2 col-form-label" aria-label="{{ _('Match Type') }}">
                     {{ _('Match Type:') }}
                 </label>
                 <select id="match-type-select" class="form-control form-control col-sm-6">
-                    <option value=""></option>
+                    <option value="">{% trans %}Default{% endtrans %}</option>
                     <option value="prefix">{% trans %}Prefix{% endtrans %}</option>
                     <option value="host">{% trans %}Host{% endtrans %}</option>
                     <option value="domain">{% trans %}Domain{% endtrans %}</option>
@@ -65,57 +85,43 @@ window.wb_prefix = "{{ wb_prefix }}";
             </div>
             <p style="cursor: help;">
                <span data-toggle="tooltip" data-placement="right"
-                     title="Restricts the results to the given date/time range (inclusive)">
+                     title="{{ _('Restricts the results to the given date/time range (inclusive)') }}">
                    {{ _('Date/Time Range') }}
                 </span>
             </p>
             <div class="form-row">
                 <div class="col-6">
-                    <label class="sr-only" for="dt-from" aria-label="Date/Time Range From">{% trans %}From:{% endtrans %}</label>
+                    <label class="sr-only" for="dt-from" aria-label="{{ _('Date/Time Range From') }}">{% trans %}From:{% endtrans %}</label>
                     <div class="input-group">
                         <div class="input-group-prepend">
                             <div class="input-group-text">{% trans %}From:{% endtrans %}</div>
                         </div>
-                        <input id="dt-from" type="number" name="date-range-from" class="form-control"
-                               pattern="^\d{4,14}$">
-                        <div class="invalid-feedback" id="dt-from-bad">
-                            {% trans %}Please enter a valid <b>From</b> timestamp. Timestamps may be 4 <= ts <=14 digits{% endtrans %}
-                        </div>
+                        <input id="dt-from" type="date" placeholder="yyyy-mm-dd" name="date-range-from" class="form-control">
+                        <input id="ts-from" type="time" placeholder="hh:mm:ss" name="date-range-from-ts" class="form-control">
                     </div>
                 </div>
                 <div class="col-6">
-                    <label class="sr-only" for="dt-to" aria-label="Date/Time Range To">{% trans %}To:{% endtrans %}</label>
+                    <label class="sr-only" for="dt-to" aria-label="{{ _('Date/Time Range To') }}">{% trans %}To:{% endtrans %}</label>
                     <div class="input-group">
                         <div class="input-group-prepend">
                             <div class="input-group-text">{% trans %}To:{% endtrans %}</div>
                         </div>
-                        <input id="dt-to" type="number" name="date-range-to" class="form-control" pattern="^\d{4,14}$">
-                        <div class="invalid-feedback" id="dt-to-bad">
-                            {% trans %}Please enter a valid <b>To</b> timestamp. Timestamps may be 4 <= ts <=14 digits{% endtrans %}
-                        </div>
+                        <input id="dt-to" type="date" placeholder="yyyy-mm-dd" name="date-range-to" class="form-control">
+                        <input id="ts-to" type="time" placeholder="hh:mm:ss" name="date-range-to-ts" class="form-control">
                     </div>
                 </div>
             </div>
             <div class="form-group mt-3">
                 <div class="form-row">
-                    <div class="col-6">
+                    <div class="col-12">
                         <p>{% trans %}Filtering{% endtrans %}</p>
                     </div>
-                    <div class="col-6">
-                        <button id="clear-filters" class="btn btn-outline-warning float-right" type="button">
-                            {% trans %}Clear Filters{% endtrans %}
-                        </button>
-                        <button id="add-filter" class="btn btn-outline-secondary float-right mr-2" type="button">
-                            {% trans %}Add Filter{% endtrans %}
-                        </button>
-                    </div>
                 </div>
                 <div class="form-row">
                     <div class="col-6">
                         <div class="row pb-1">
                             <label for="filter-by" class="col-form-label col-3">{% trans %}By:{% endtrans %}</label>
                             <select id="filter-by" class="form-control col-7">
-                                <option value="" selected></option>
                                 <option value="mime">{% trans %}Mime Type{% endtrans %}</option>
                                 <option value="status">{% trans %}Status{% endtrans %}</option>
                                 <option value="url">{% trans %}URL{% endtrans %}</option>
@@ -129,20 +135,27 @@ window.wb_prefix = "{{ wb_prefix }}";
                                 <option value="=~">{% trans %}Matches Regex{% endtrans %}</option>
                                 <option value="=!">{% trans %}Does Not Contain{% endtrans %}</option>
                                 <option value="=!=">{% trans %}Is Not{% endtrans %}</option>
-                                <option value="=!~">{% trans %}Does Not Begins With{% endtrans %}</option>
+                                <option value="=!~">{% trans %}Does Not Begin With{% endtrans %}</option>
                             </select>
                         </div>
-                        <div class="row">
+                        <div class="row pb-1">
                             <label for="filter-expression" class="col-form-label col-3">{% trans %}Expr:{% endtrans %}</label>
                             <input type="text" id="filter-expression" class="form-control col-7"
-                                   placeholder="Enter an expression to filter by"
+                                   placeholder="{% trans %}Enter an expression to filter by{% endtrans %}"
                             >
                         </div>
+                            <button id="add-filter" class="btn btn-outline-secondary mt-2" type="button">
+                                {% trans %}Add Filter{% endtrans %}
+                            </button>
+
                     </div>
                     <div class="col-6">
                         <ul id="filter-list" class="filter-list">
                             <li id="filtering-nothing">{% trans %}No Filter{% endtrans %}</li>
                         </ul>
+                        <button id="clear-filters" class="btn btn-outline-danger float-right mr-2" type="button">
+                            {% trans %}Clear Filters{% endtrans %}
+                        </button>
                     </div>
                 </div>
             </div>
@@ -170,7 +183,7 @@ window.wb_prefix = "{{ wb_prefix }}";
                              id="collection-metadata-{{ key }}" role="tabpanel">
                             <div class="card inherit-height">
                                 <div class="card-body">
-                                    {{ val }}
+                                    {{ val | safe }}
                                 </div>
                             </div>
                         </div>
@@ -180,4 +193,5 @@ window.wb_prefix = "{{ wb_prefix }}";
         </div>
     </div>
 {% endif %}
+{% include "instructions.html" %}
 {% endblock %}

pywb/templates/vue_loc.html

@@ -0,0 +1,68 @@
+<script>
+
+{% autoescape false %}
+    var allLocales =  {{ get_locale_prefixes() }};
+{% endautoescape %}
+    var i18nStrings = {
+        jan_long: "{{ _Q('January') }}",
+        feb_long: "{{ _Q('February') }}",
+        mar_long: "{{ _Q('March') }}",
+        apr_long: "{{ _Q('April') }}",
+        may_long: "{{ _Q('May') }}",
+        jun_long: "{{ _Q('June') }}",
+        jul_long: "{{ _Q('July') }}",
+        aug_long: "{{ _Q('August') }}",
+        sep_long: "{{ _Q('September') }}",
+        oct_long: "{{ _Q('October') }}",
+        nov_long: "{{ _Q('November') }}",
+        dec_long: "{{ _Q('December') }}",
+        jan_short: "{{ _Q('Jan') }}",
+        feb_short: "{{ _Q('Feb') }}",
+        mar_short: "{{ _Q('Mar') }}",
+        apr_short: "{{ _Q('Apr') }}",
+        may_short: "{{ _Q('May') }}",
+        jun_short: "{{ _Q('Jun') }}",
+        jul_short: "{{ _Q('Jul') }}",
+        aug_short: "{{ _Q('Aug') }}",
+        sep_short: "{{ _Q('Sep') }}",
+        oct_short: "{{ _Q('Oct') }}",
+        nov_short: "{{ _Q('Nov') }}",
+        dec_short: "{{ _Q('Dec') }}",
+        mon_short: "{{ _Q('Mon') }}",
+        tue_short: "{{ _Q('Tue') }}",
+        wed_short: "{{ _Q('Wed') }}",
+        thu_short: "{{ _Q('Thu') }}",
+        fri_short: "{{ _Q('Fri') }}",
+        sat_short: "{{ _Q('Sat') }}",
+        sun_short: "{{ _Q('Sun') }}",
+        mon_long: "{{ _Q('Monday') }}",
+        tue_long: "{{ _Q('Tuesday') }}",
+        wed_long: "{{ _Q('Wednesday') }}",
+        thu_long: "{{ _Q('Thursday') }}",
+        fri_long: "{{ _Q('Friday') }}",
+        sat_long: "{{ _Q('Saturday') }}",
+        sun_long: "{{ _Q('Sunday') }}",
+        "All-time": "{{ _Q('All-time') }}",
+        "Show timeline":"{{ _Q('Show timeline') }}",
+        "Hide timeline":"{{ _Q('Hide timeline') }}",
+        "Show calendar":"{{ _Q('Show calendar') }}",
+        "Hide calendar":"{{ _Q('Hide calendar') }}",
+        "Previous capture":"{{ _Q('Previous capture') }}",
+        "Next capture":"{{ _Q('Next capture') }}",
+        "Print":"{{ _Q('Print') }}",
+        "Select language":"{{ _Q('Select language') }}",
+        "View capture on {date}":"{{ _Q('View capture on {date}') }}",
+        "{count} capture":"{{ _Q('{count} capture') }}",
+        "{count} captures":"{{ _Q('{count} captures') }}",
+        "{capture_text} on {date}":"{{ _Q('{capture_text} on {date}') }}",
+        "{capture_text} in {month}":"{{ _Q('{capture_text} in {month}') }}",
+        "current":"{{ _Q('current') }}", // translators: current capture in list of captures
+        "Loading...": "{{ _Q('Loading...') }}",
+        "Current Capture": "{{ _Q('Current Capture') }}",
+        "capture": "{{ _Q('capture') }}",
+        "captures": "{{ _Q('captures') }}",
+        "from {hour1} to {hour2}": "{{ _Q('from {hour1} to {hour2}') }}",
+        "no captures": "{{ _Q('no captures') }}",
+        "Archived Page: ": "{{ _Q('Archived Page: ') }}"
+      }
+</script>

pywb/utils/binsearch.py

@@ -150,7 +150,7 @@ def iter_exact(reader, key, token=b' '):
     """
     Create an iterator which iterates over lines where the first field matches
     the 'key', equivalent to token + sep prefix.
-    Default field termin_ator/seperator is ' '
+    Default field termin_ator/separator is ' '
     """
 
     return iter_prefix(reader, key + token)

pywb/utils/loaders.py

@@ -185,7 +185,8 @@ class BlockLoader(BaseLoader):
     """
     a loader which can stream blocks of content
     given a uri, offset and optional length.
-    Currently supports: http/https and file/local file system
+    Currently supports: http/https, file/local file system,
+    pkg, WebHDFS, S3
     """
 
     loaders = {}
@@ -393,14 +394,15 @@ class S3Loader(BaseLoader):
 
         def s3_load(anon=False):
             if not self.client:
+                s3_client_args = {}
                 if anon:
-                    config = Config(signature_version=UNSIGNED)
-                else:
-                    config = None
+                    s3_client_args['config'] = Config(signature_version=UNSIGNED)
+                if aws_access_key_id:
+                    s3_client_args['aws_access_key_id'] = aws_access_key_id
+                    s3_client_args['aws_secret_access_key'] = aws_secret_access_key
+
+                client = boto3.client('s3', **s3_client_args)
 
-                client = boto3.client('s3', aws_access_key_id=aws_access_key_id,
-                                      aws_secret_access_key=aws_secret_access_key,
-                                      config=config)
             else:
                 client = self.client
 

pywb/utils/test/test_loaders.py

@@ -97,10 +97,19 @@ from pywb import get_test_dir
 
 test_cdx_dir = get_test_dir() + 'cdx/'
 
+def s3_authenticated_access_verification(bucket):
+    import boto3, botocore
+    s3_client = boto3.client('s3')
+    try:
+        s3_client.head_bucket(Bucket=bucket)
+    except botocore.exceptions.NoCredentialsError:
+        pytest.skip("Skipping S3Loader test for authenticated reads: no credentials configured")
 
-def test_s3_read_1():
+def test_s3_read_authenticated_1():
     pytest.importorskip('boto3')
 
+    s3_authenticated_access_verification('commoncrawl')
+
     res = BlockLoader().load('s3://commoncrawl/crawl-data/CC-MAIN-2015-11/segments/1424936462700.28/warc/CC-MAIN-20150226074102-00159-ip-10-28-5-156.ec2.internal.warc.gz',
                              offset=53235662,
                              length=2526)
@@ -112,13 +121,15 @@ def test_s3_read_1():
     assert reader.readline() == b'WARC/1.0\r\n'
     assert reader.readline() == b'WARC-Type: response\r\n'
 
-def test_s3_read_2():
+def test_s3_read_authenticated_2():
     pytest.importorskip('boto3')
 
+    s3_authenticated_access_verification('commoncrawl')
+
     res = BlockLoader().load('s3://commoncrawl/crawl-data/CC-MAIN-2015-11/index.html')
 
     buff = res.read()
-    assert len(buff) == 2082
+    assert len(buff) == 2330
 
     reader = DecompressingBufferedReader(BytesIO(buff))
     assert reader.readline() == b'<!DOCTYPE html>\n'

pywb/version.py

@@ -1,4 +1,4 @@
-__version__ = '2.6.4'
+__version__ = '2.8.3'
 
 if __name__ == '__main__':
     print(__version__)

pywb/vueui/.eslintrc.js

@@ -0,0 +1,39 @@
+module.exports = {
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:vue/essential"
+    ],
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "vue"
+    ],
+    "rules": {
+        "no-restricted-globals": [
+            2,
+            "event", "error"
+        ],
+        "indent": [
+            "error",
+            2
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "double"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ]
+    }
+};

pywb/vueui/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "typescript",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "build": "rollup -c",
+    "lint": "eslint ./src/*.js ./src/*.vue ./src/components/*.vue"
+  },
+  "dependencies": {
+    "vue": "^2.6.11",
+    "vue-template-compiler": "^2.6.14"
+  },
+  "devDependencies": {
+    "@rollup/plugin-node-resolve": "^13.0.4",
+    "eslint": "^7.32.0",
+    "eslint-plugin-vue": "^7.17.0",
+    "rollup": "^2.10.9",
+    "rollup-plugin-css-only": "^3.1.0",
+    "rollup-plugin-vue": "^5.0.0"
+  }
+}

pywb/vueui/rollup.config.js

@@ -0,0 +1,20 @@
+import vue from "rollup-plugin-vue";
+import css from "rollup-plugin-css-only";
+import { nodeResolve } from "@rollup/plugin-node-resolve";
+
+export default [
+  {
+    input: "src/index.js",
+    output: {
+      file: "../static/vue/vueui.js",
+      sourcemap: "inline",
+      name: "VueUI",
+      format: "iife",
+    },
+    plugins: [
+      vue({css: true, compileTemplate: true}),
+      css(),
+      nodeResolve({browser: true})
+    ],
+  },
+];

pywb/vueui/src/App.vue

@@ -0,0 +1,515 @@
+<template>
+  <div class="app" :class="{expanded: showTimelineView || showFullView }" data-app="webrecorder-replay-app">
+    <!-- Top navbar -->
+    <nav
+      class="navbar navbar-light navbar-expand-lg fixed-top top-navbar justify-content-center"
+      :style="navbarStyle">
+      <a class="navbar-brand flex-grow-1 my-1" :href="config.logoHomeUrl" v-if="config.logoHomeUrl">
+        <img :src="config.logoImg" id="logo-img" alt="_('pywb logo')">
+      </a>
+      <div class="navbar-brand flex-grow-1 my-1" v-else>
+        <img :src="config.logoImg" id="logo-img" alt="_('pywb logo')">
+      </div>
+      <div class="flex-grow-1 d-flex" id="searchdiv">
+        <form
+          class="form-inline my-2 my-md-0 mx-lg-auto"
+          role="search"
+          @submit="gotoUrl">
+          <input
+            id="theurl"
+            type="text"
+            :value="config.url"
+            height="31"
+            aria-label="_('Search for archival capture of URL')"
+            title="_('Search for archival capture of URL')"></input>
+        </form>
+      </div>
+      <button
+        class="navbar-toggler btn btn-sm"
+        id="collapse-button"
+        type="button"
+        data-toggle="collapse"
+        data-target="#navbarCollapse"
+        aria-controls="navbarCollapse"
+        aria-expanded="false"
+        aria-label="_('Toggle navigation')">
+        <span class="navbar-toggler-icon"></span>
+      </button>
+      <div class="collapse navbar-collapse ml-auto" id="navbarCollapse">
+        <ul class="navbar-nav ml-3" id="toggles">
+          <li class="nav-item">
+            <button
+              class="btn btn-sm"
+              :class="{active: showFullView, 'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              :title="_('Previous capture')"
+              v-if="previousSnapshot"
+              @click="gotoPreviousSnapshot">
+              <i class="fas fa-arrow-left" :title="_('Previous capture')"></i>
+            </button>
+          </li>
+          <li class="nav-item">
+            <button
+              class="btn btn-sm"
+              :class="{active: showFullView, 'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              :title="_('Next capture')"
+              v-if="nextSnapshot"
+              @click="gotoNextSnapshot">
+              <i class="fas fa-arrow-right" :title="_('Next capture')"></i>
+            </button>
+          </li>
+          <li class="nav-item active">
+            <button
+              class="btn btn-sm"
+              :class="{active: showFullView, 'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              :aria-pressed="(showFullView ? true : false)"
+              @click="showFullView = !showFullView"
+              :title="(showFullView ? _('Hide calendar') : _('Show calendar'))">
+              <i class="far fa-calendar-alt"></i>
+            </button>
+          </li>
+          <li class="nav-item">
+            <button
+              class="btn btn-sm"
+              :class="{active: showTimelineView, 'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              :aria-pressed="showTimelineView"
+              @click="toggleTimelineView"
+              :title="(showTimelineView ? _('Hide timeline') : _('Show timeline'))">
+              <i class="far fa-chart-bar"></i>
+            </button>
+          </li>
+          <li class="nav-item">
+            <button
+              class="btn btn-sm"
+              :class="{'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              :aria-pressed="printReplayFrame"
+              @click="printReplayFrame"
+              v-if="printingEnabled && hasReplayFrame()"
+              :title="_('Print')">
+              <i class="fas fa-print"></i>
+            </button>
+          </li>
+          <li class="nav-item dropdown" v-if="localesAreSet">
+            <button
+              class="btn btn-sm dropdown-toggle"
+              :class="{'btn-outline-light': lightButtons, 'btn-outline-dark': !lightButtons}"
+              type="button"
+              id="locale-dropdown"
+              data-toggle="dropdown"
+              aria-haspopup="true"
+              aria-expanded="false"
+              :title="_('Select language')">
+              <i class="fas fa-globe-africa" :title="_('Language')"></i>
+            </button>
+            <div class="dropdown-menu dropdown-menu-right" aria-labelledby="locale-dropdown">
+              <a
+                class="dropdown-item"
+                v-for="(locPath, key) in config.allLocales"
+                :key="key"
+                :href="locPath + (currentSnapshot ? currentSnapshot.id : '*') + '/' + config.url">
+                {{ key }}
+              </a>
+            </div>
+          </li>
+        </ul>
+      </div>
+    </nav>
+
+    <!-- Capture title and date -->
+    <nav
+      class="navbar navbar-light justify-content-center title-nav fixed-top"
+      id="second-navbar"
+      :style="navbarStyle">
+      <span class="hidden" v-if="!currentSnapshot">&nbsp;</span>
+      <span v-if="currentSnapshot">
+        <span class="strong mr-1">
+          {{_('Current Capture')}}: 
+          <span class="ml-1" v-if="config.title">
+            {{ config.title }}
+          </span>
+        </span>
+        <span class="mr-1" v-if="config.title">|</span>
+        {{currentSnapshot.getTimeDateFormatted()}}
+      </span>
+    </nav>
+
+    <!-- Timeline -->
+    <div class="card border-top-0 border-left-0 border-right-0 timeline-wrap">
+      <div class="card-body" v-if="currentPeriod && showTimelineView">
+        <div class="row">
+          <div class="col col-12">
+            <TimelineBreadcrumbs
+              :period="currentPeriod"
+              @goto-period="gotoPeriod"
+            ></TimelineBreadcrumbs>
+          </div>
+          <div class="col col-12 mt-2">
+            <Timeline
+              :period="currentPeriod"
+              :highlight="timelineHighlight"
+              :current-snapshot="currentSnapshot"
+              :max-zoom-level="maxTimelineZoomLevel"
+              @goto-period="gotoPeriod"
+            ></Timeline>
+          </div>
+        </div>
+      </div>    
+    </div>
+
+    <!-- Calendar -->
+    <div class="card" id="calendar-card" v-if="currentPeriod && showFullView && currentPeriod.children.length">
+      <div class="card-body" id="calendar-card-body">
+        <CalendarYear
+          :period="currentPeriod"
+          :current-snapshot="currentSnapshot"
+           @goto-period="gotoPeriod">
+        </CalendarYear>
+      </div>
+    </div>
+    
+  </div>
+</template>
+
+<script>
+import Timeline from "./components/Timeline.vue";
+import TimelineBreadcrumbs from "./components/TimelineBreadcrumbs.vue";
+import CalendarYear from "./components/CalendarYear.vue";
+
+import { PywbSnapshot, PywbPeriod } from "./model.js";
+import {PywbI18N} from "./i18n";
+
+export default {
+  name: "PywbReplayApp",
+  //el: '[data-app="webrecorder-replay-app"]',
+  data: function() {
+    return {
+      snapshots: [],
+      currentPeriod: null,
+      currentSnapshot: null,
+      currentSnapshotIndex: null,
+      msgs: [],
+      showFullView: false,
+      showTimelineView: false,
+      maxTimelineZoomLevel: PywbPeriod.Type.day,
+      config: {
+        title: "",
+        initialView: {},
+        allLocales: {}
+      },
+      timelineHighlight: false,
+      locales: [],
+    };
+  },
+  components: {Timeline, TimelineBreadcrumbs, CalendarYear},
+  mounted: function() {
+    // add empty unload event listener to make this page bfcache ineligible.
+    // bfcache otherwises prevent the query template from reloading as expected
+    // when the user navigates there via browser back/forward buttons
+    addEventListener('unload', (event) => { });
+  },
+  updated: function() {
+    // set top frame title equal to value pulled from replay frame
+    document.title = this._("Archived Page: ") + this.config.title;
+  },
+  computed: {
+    sessionStorageUrlKey() {
+      // remove http(s), www and trailing slash
+      return 'zoom__' + this.config.url.replace(/^https?:\/\/(www\.)?/, '').replace(/\/$/, '');
+    },
+    localesAreSet() {
+      return Object.entries(this.config.allLocales).length > 0;
+    },
+    navbarStyle() {
+      return {
+        '--navbar-background': `#${this.config.navbarBackground}`,
+        '--navbar-color': `#${this.config.navbarColor}`
+      }
+    },
+    lightButtons() {
+      return !!this.config.navbarLightButtons;
+    },
+    printingEnabled() {
+      return !this.config.disablePrinting;
+    },
+    previousSnapshot() {
+      if (!this.currentSnapshotIndex) {
+        return null;
+      }
+      if (this.currentSnapshotIndex > 0) {
+        return this.snapshots[this.currentSnapshotIndex - 1];
+      }
+      return null;
+    },
+    nextSnapshot() {
+      if (this.currentSnapshotIndex == null) {
+        return null;
+      }
+      if (
+        (this.currentSnapshotIndex >= 0)
+        && (this.currentSnapshotIndex !== this.snapshots.length - 1)) {
+        return this.snapshots[this.currentSnapshotIndex + 1];
+      }
+      return null;
+    }
+  },
+  methods: {
+    _(id, embeddedVariableStrings=null) {
+      return PywbI18N.instance.getText(id, embeddedVariableStrings);
+    },
+    gotoPeriod: function(newPeriod, onlyZoomToPeriod) {
+      if (this.timelineHighlight) {
+        setTimeout((() => {
+          this.timelineHighlight=false;
+        }).bind(this), 3000);
+      }
+      // only go to snapshot if caller did not request to zoom only
+      if (newPeriod.snapshot && !onlyZoomToPeriod) {
+        this.gotoSnapshot(newPeriod.snapshot, newPeriod, true /* reloadIFrame */);
+      } else {
+        // save current period (aka zoom)
+        // use sessionStorage (not localStorage), as we want this to be a very temporary memory for current page tab/window and no longer; NOTE: it serves when navigating from an "*" query to a specific capture and subsequent reloads
+        if (window.sessionStorage) {
+          window.sessionStorage.setItem(this.sessionStorageUrlKey, newPeriod.fullId);
+        }
+        // If new period goes beyond allowed max level
+        if (newPeriod.type > this.maxTimelineZoomLevel) {
+          this.currentPeriod = newPeriod.get(this.maxTimelineZoomLevel);
+        } else {
+          this.currentPeriod = newPeriod;
+        }
+      }
+    },
+    gotoSnapshot(snapshot, fromPeriod, reloadIFrame=false) {
+      this.currentSnapshot = snapshot;
+
+      const isCurrentSnapshot = (snapshotInArray) => snapshotInArray.id == snapshot.id && snapshotInArray.url == snapshot.url;
+      this.currentSnapshotIndex = this.snapshots.findIndex(isCurrentSnapshot);
+
+      // if the current period doesn't match the current snapshot, update it
+      if (!this.currentPeriod || (fromPeriod && !this.currentPeriod.contains(fromPeriod))) {
+        const fromPeriodAtMaxZoomLevel = fromPeriod.get(this.maxTimelineZoomLevel);
+        if (!this.currentPeriod || fromPeriodAtMaxZoomLevel !== this.currentPeriod) {
+          this.currentPeriod = fromPeriodAtMaxZoomLevel;
+        }
+      }
+
+      // update iframe only if the snapshot was selected from the calendar/timeline.
+      // if the change originated from a user clicking a link in the iframe, emitting
+      // snow-shapshot will only cause a flash of content
+      if (reloadIFrame !== false) {
+        this.$emit("show-snapshot", snapshot);
+      }
+      this.initBannerState(true);
+    },
+    gotoPreviousSnapshot() {
+      let periodToChangeTo = this.currentPeriod.findByFullId(this.previousSnapshot.getFullId());
+      this.gotoPeriod(periodToChangeTo, false /* onlyZoomToPeriod */);
+    },
+    gotoNextSnapshot() {
+      let periodToChangeTo = this.currentPeriod.findByFullId(this.nextSnapshot.getFullId());
+      this.gotoPeriod(periodToChangeTo, false /* onlyZoomToPeriod */);
+    },
+    gotoUrl(event) {
+      event.preventDefault();
+      const newUrl = document.querySelector("#theurl").value;
+      if (newUrl !== this.config.url) {
+        const ts = this.config.timestamp === undefined ? "*" : this.config.timestamp;
+        window.location.href = this.config.prefix + ts + (ts ? "/" : "") + newUrl;
+      }
+    },
+    toggleTimelineView() {
+      this.showTimelineView = !this.showTimelineView;
+      window.localStorage.setItem("showTimelineView", this.showTimelineView ? "1" : "0");
+    },
+    hasReplayFrame() {
+      return !! window.frames.replay_iframe;
+    },
+    printReplayFrame() {
+      window.frames.replay_iframe.contentWindow.focus();
+      window.frames.replay_iframe.contentWindow.print();
+      return false;
+    },
+    setData(/** @type {PywbData} data */ data) {
+
+      // data-set will usually happen at App INIT (from parent caller)
+      this.$set(this, "snapshots", data.snapshots);
+      this.$set(this, "currentPeriod", data.timeline);
+
+      // get last-saved current period from previous page/app refresh (if there was such)
+      if (window.sessionStorage) {
+        const currentPeriodId = window.sessionStorage.getItem(this.sessionStorageUrlKey);
+        if (currentPeriodId) {
+          const newCurrentPeriodFromStorage = this.currentPeriod.findByFullId(currentPeriodId);
+          if (newCurrentPeriodFromStorage) {
+            this.currentPeriod = newCurrentPeriodFromStorage;
+          }
+        }
+      }
+
+      // signal app is DONE setting and rendering data; ON NEXT TICK
+      this.$nextTick(function isDone() {
+        this.$emit('data-set-and-render-completed');
+      }.bind(this));
+    },
+    setSnapshot(view) {
+      if (!this.currentPeriod) {
+        return false;
+      }
+
+      // turn off calendar (aka full) view
+      this.showFullView = false;
+
+      // convert to snapshot object to support proper rendering of time/date
+      const snapshot = new PywbSnapshot(view, 0);
+
+      this.config.url = view.url;
+
+      let periodToChangeTo = this.currentPeriod.findByFullId(snapshot.getFullId());
+      if (periodToChangeTo) {
+        this.gotoPeriod(periodToChangeTo, false /* onlyZoomToPeriod */);
+        return true;
+      }
+      return false;
+    },
+    initBannerState(isReplay) {
+      // if not replay, always show both
+      if (!isReplay) {
+        this.showFullView = true;
+        this.showTimelineView = true;
+      } else {
+        this.showFullView = false;
+        this.showTimelineView = window.localStorage.getItem("showTimelineView") === "1";
+      }
+    },
+    updateTitle(title) {
+      this.config.title = title;
+    }
+  }
+};
+</script>
+
+<style>
+  body {
+    padding-top: 89px !important;
+  }
+  .app {
+    font-family: Calibri, Arial, sans-serif;
+    /*border-bottom: 1px solid lightcoral;*/
+    width: 100%;
+  }
+  .app.expanded {
+    /*height: 130px;*/
+    max-height: calc(100vh - 90px);
+    display: flex;
+    flex-direction: column;
+  }
+  .full-view {
+    /*position: fixed;*/
+    /*top: 150px;*/
+    left: 0;
+  }
+  .navbar {
+    background-color: var(--navbar-background);
+    color:  var(--navbar-color);
+  }
+  .top-navbar {
+    z-index: 90;
+    padding: 2px 16px 0 16px;
+  }
+  .top-navbar span.navbar-toggler-icon {
+    margin: .25rem !important;
+  }
+  #logo-img {
+    max-height: 40px;
+  }
+  .title-nav {
+    margin-top: 50px;
+    z-index: 80;
+  }
+  #secondNavbar {
+    height: 24px !important;
+  }
+  #navbarCollapse {
+    justify-content: right;
+  }
+  #navbarCollapse ul#toggles {
+    display: flex;
+    align-content: center;
+  }
+  #navbarCollapse:not(.show) ul#toggles li:not(:first-child) {
+    margin-left: .25rem;
+  }
+  #navbarCollapse.show {
+    padding-bottom: 1em;
+  }
+  #navbarCollapse.show ul#toggles li {
+    margin-top: 5px;
+  }
+  #navbarCollapse.show ul#toggles li {
+    margin-left: 0px;
+  }
+  .iframe iframe {
+    width: 100%;
+    height: 80vh;
+  }
+  #searchdiv {
+    height: 31px;
+  }
+  #theurl {
+    width: 250px;
+  }
+  @media (min-width: 576px) {
+    #theurl {
+      width: 350px;
+    }
+  }
+  @media (min-width: 768px) {
+    #theurl {
+      width: 500px;
+    }
+  }
+  @media (min-width: 992px) {
+    #theurl {
+      width: 600px;
+    }
+  }
+  @media (min-width: 1200px) {
+    #theurl {
+      width: 900px;
+    }
+  }
+  #toggles {
+    align-items: center;
+  }
+  .breadcrumb-row {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+  }
+  div.timeline-wrap div.card {
+    margin-top: 55px;
+  }
+  #calendar-card {
+    overflow-y: auto;
+    max-height: 100%;
+  }
+  div.timeline-wrap div.card-body {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+  }
+  div.timeline-wrap div.card-body div.row {
+    width: 100%;
+    align-items: center;
+    justify-content: center;
+  }
+
+  #calendar-card-body {
+    padding: 0;
+  }
+  .strong {
+    font-weight: bold;
+  }
+  .hidden {
+    color: var(--navbar-background);
+  }
+</style>

pywb/vueui/src/cdx-simulator/README.md

@@ -0,0 +1,41 @@
+# How to incorporate CDX Simulator
+
+Place following code snippets in **index.js**
+
+## Import  `CDXQueryWorkerSimulator` Mock Class
+
+It is the mock class to the main javascript built-in `Worker` class:
+
+   ``import { CDXQueryWorkerSimulator } from "./cdx-simulator/cdx-simulator";``
+
+## Initialize `queryWorker` with Mock Class
+
+Update `const queryWorker = ...` initialization in `CDXLoader` class, `loadCDX()` method
+   
+### by replacing it
+
+```
+const queryWorker = new CDXQueryWorkerSimulator(this.staticPrefix + "/queryWorker.js");
+```
+
+### or by adding a conditional, so you can go back and forth between simulator and real CDX-data-loader:
+
+for example with a URL-hash flag conditional:
+
+```
+const queryWorker = new (window.location.hash.indexOf('cdx_simulate') >= 0 ? CDXQueryWorkerSimulator : Worker)(this.staticPrefix + "/queryWorker.js");
+```
+
+NOTE: where if the url contains '#cdx_simulate' the mock simulator will be used; using a URL hash does not interfere with the main URL parsing of the PYWB app
+
+## Configure Simulation 
+
+Add a **local** storage (from Chrome Dev Tools > Application > Local Storage)
+
+```
+{"count":5000, "yearStart":2020, "yearEnd":2022, "fetchTime":3000}
+```
+
+where `count` is the total records, yearStart and yearEnd are self-explanatory, and `fetchTime` is how long it should take
+
+![cdx loader config](pywb-vueui-cdx-simulator-config.jpg)

pywb/vueui/src/cdx-simulator/cdx-record-sample.json

@@ -0,0 +1,197 @@
+[
+  {
+    "urlkey": "com,example)/",
+    "timestamp": "20130729195151",
+    "url": "http://test@example.com/",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "591",
+    "offset": "355",
+    "filename": "example-url-agnostic-revisit.warc.gz",
+    "source": "pywb:url-agnost-example.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/",
+    "timestamp": "20140127171200",
+    "url": "http://example.com",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1046",
+    "offset": "334",
+    "filename": "dupes.warc.gz",
+    "source": "pywb:dupes.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/",
+    "timestamp": "20140127171251",
+    "url": "http://example.com",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "553",
+    "offset": "11875",
+    "filename": "dupes.warc.gz",
+    "source": "pywb:dupes.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=1",
+    "timestamp": "20140103030321",
+    "url": "http://example.com?example=1",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1043",
+    "offset": "333",
+    "filename": "example.warc.gz",
+    "source": "pywb:example.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=1",
+    "timestamp": "20140103030341",
+    "url": "http://example.com?example=1",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "553",
+    "offset": "1864",
+    "filename": "example.warc.gz",
+    "source": "pywb:example.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=2",
+    "timestamp": "20140103030321",
+    "url": "http://example.com?example=2",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1987",
+    "offset": "0",
+    "filename": "example-extra.warc",
+    "source": "pywb:example-extra.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=2",
+    "timestamp": "20140603030341",
+    "url": "http://example.com?example=2",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "504",
+    "offset": "2701",
+    "filename": "example-extra.warc",
+    "source": "pywb:example-extra.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=2",
+    "timestamp": "20140603030351",
+    "url": "http://example.com?example=2",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36B",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "504",
+    "offset": "2701",
+    "filename": "example-extra.warc",
+    "source": "pywb:bad.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=2",
+    "timestamp": "20140703030321",
+    "url": "http://example.com?example=2",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1987",
+    "offset": "0",
+    "filename": "non-existent.warc",
+    "source": "pywb:bad.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=3",
+    "timestamp": "20140603030351",
+    "url": "http://example.com?example=3",
+    "mime": "warc/revisit",
+    "status": "-",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36B",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "504",
+    "offset": "2701",
+    "filename": "example-extra.warc",
+    "source": "pywb:bad.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example)/?example=3",
+    "timestamp": "20140703030321",
+    "url": "http://example.com?example=3",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1987",
+    "offset": "0",
+    "filename": "non-existent.warc",
+    "source": "pywb:bad.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example,test,arc)/",
+    "timestamp": "20140216050221",
+    "url": "http://example.com/",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "1656",
+    "offset": "151",
+    "filename": "example.arc",
+    "source": "pywb:example-arc-test.cdx",
+    "source-coll": "pywb"
+  },
+  {
+    "urlkey": "com,example,test,gz,arc)/",
+    "timestamp": "20140216050221",
+    "url": "http://example.com/",
+    "mime": "text/html",
+    "status": "200",
+    "digest": "B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A",
+    "redirect": "-",
+    "robotflags": "-",
+    "length": "856",
+    "offset": "171",
+    "filename": "example.arc.gz",
+    "source": "pywb:example-arc-test.cdx",
+    "source-coll": "pywb"
+  }
+]

pywb/vueui/src/cdx-simulator/cdx-simulator.js

@@ -0,0 +1,83 @@
+const getMonthDays = (y, mZeroIndex) => {
+  const firstOfNextMonth = new Date(y, mZeroIndex+1, 1);
+  const lastOfMonth = new Date(firstOfNextMonth - 1000 * 3600 * 24);
+  return lastOfMonth.getDate();
+}
+
+// read dynamically from local storage options for make
+let simulateCdxOptions = window.localStorage.getItem('cdx_simulate');
+simulateCdxOptions = !!simulateCdxOptions ? JSON.parse(simulateCdxOptions) : {};
+
+class CDXRecordFactory {
+  constructor() {}
+
+  async make(url, opts={}) {
+    // defaults
+    opts = {count:1000, yearStart:2015, yearEnd:2022, fetchTime:5*1000, ...opts};
+
+    const records = [];
+
+    const total = opts.count;
+    const years = [opts.yearStart, opts.yearEnd];
+    const avgPerMonth = total / (years[1]-years[0]) / 12;
+    // exaggerate max count per day, any day can hold up to 10th of the month's captures
+    const maxPerDay = avgPerMonth/10;
+
+    let startTime = Math.floor(new Date().getTime());
+    let recordI = 0;
+
+    for(let y=years[0]; y<=years[1]; y++) {
+      for(let m=1; m<=12; m++) {
+        for(let d=1; d<=getMonthDays(y, m-1); d++) {
+          const dayTimestampPrefix = y + ('0'+m).substr(-2) + ('0'+d).substr(-2);
+          // minumum to maximum count (random value)
+          const timesCount = Math.floor(Math.random() * maxPerDay);
+
+          const times = {}; // make sure we save to hash to de-dupe
+          for(let i=0; i<timesCount; i++) {
+            const newTime = [Math.floor(Math.random()*24), Math.floor(Math.random()*60), Math.floor(Math.random()*60)].join('');
+            times[newTime] = 1;
+          }
+          Object.keys(times).sort().forEach(time => {
+            records.push({url, timestamp: dayTimestampPrefix+time});
+          });
+        }
+      }
+    }
+    let endTime = Math.floor(new Date().getTime());
+
+    if (opts.fetchTime && opts.fetchTime > endTime - startTime) { // wait till we reac fetchTime
+      const p = new Promise((resolve) => {
+        setTimeout(() => {
+          resolve(true);
+        }, (opts.fetchTime - (endTime - startTime)));
+      });
+      await p;
+    }
+    return records;
+  }
+}
+
+export class CDXQueryWorkerSimulator {
+  constructor(workerPath) {
+    this.messageCb = [];
+    this.recordFactory = new CDXRecordFactory();
+  }
+
+
+  addEventListener(type, cb) {
+    if (type === 'message') {
+      this.messageCb = cb;
+    }
+  }
+
+  async postMessage({type, queryUrl}) {
+    const records = await this.recordFactory.make(queryUrl, simulateCdxOptions);
+    records.forEach(record => this.messageCb({data: {type: 'cdxRecord', record}}));
+    this.messageCb({data: {type: 'finished'}});
+  }
+
+  terminate() {
+    return true;
+  }
+}

pywb/vueui/src/cdx-simulator/test.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>CDX Simulator</title>
+    <script src="cdx-simulator.js"></script>
+</head>
+<body>
+<script>
+    const factory = new CDXRecordFactory();
+    console.log(factory.make('test.com'));
+</script>
+</body>
+</html>

pywb/vueui/src/components/CalendarMonth.vue

@@ -0,0 +1,182 @@
+<style>
+    .calendar-month {
+        position: relative;
+        display: inline-block;
+        padding: 5px;
+        margin: 0;
+        height: 260px;
+        width: 220px;
+        text-align: center;
+        vertical-align: top;
+        box-sizing: content-box;
+        border-radius: 10px;
+    }
+    .calendar-month:hover {
+        background-color: #F5F5F5;
+    }
+    .calendar-month.current {
+        background-color: #fff7ce;
+    }
+    .calendar-month.contains-current-snapshot {
+        border: solid 1px red;
+    }
+    .calendar-month > h3 {
+        margin: 0;
+        font-size: 16px;
+    }
+    .calendar-month > .empty {
+        position: absolute;
+        top: 45%;
+        width: 100%;
+        color: #454545;
+    }
+    .calendar-month .day {
+        position: relative;
+        display: inline-block;
+        margin: 0;
+        text-align: center;
+    }
+    .calendar-month .day.empty {
+        color: #454545;
+    }
+    .calendar-month .day .count {
+        display: none;
+        position: absolute;
+        bottom: 80%;
+        left: 80%;
+        line-height: 1; /* reset to normal */
+        padding: 3px;
+        border-radius: 10px;
+        border-bottom-left-radius: 0;
+        border: 1px solid gray;
+        background-color: white;
+        z-index: 30;
+        white-space: nowrap;
+    }
+    .calendar-month .day:hover .count {
+        display: block;
+    }
+    .calendar-month .day .size {
+        position: absolute;
+        box-sizing: border-box;
+        background-color: rgba(166, 205, 245, .85);
+        z-index: 10;
+    }
+    .calendar-month .day.contains-current-snapshot .size {
+      background-color: rgba(255, 100, 100, .85);
+    }
+
+    .calendar-month .day .day-id {
+        position: absolute;
+        top: 0;
+        left: 0;
+        z-index: 11;
+
+        display: inline-block;
+        width: 100%;
+        text-align: center;
+
+        color: black;
+    }
+    .calendar-month .day .day-id.empty {
+      color: #454545 !important;
+    }
+    .calendar-month .day:hover .size {
+        border: 1px solid black;
+    }
+    .calendar-month .day:hover {
+        cursor: zoom-in;
+    }
+    .calendar-month .day.empty:hover {
+        cursor: not-allowed;
+    }
+</style>
+
+<template>
+  <div class="calendar-month" :class="{current: isCurrent, 'contains-current-snapshot': containsCurrentSnapshot}">
+    <h3>{{getLongMonthName(month.id)}} <span v-if="month.snapshotCount">({{ month.snapshotCount }})</span></h3>
+    <span v-for="(dayInitial) in dayInitials" class="day" :style="dayStyle">{{dayInitial}}</span><br/>
+    <span v-for="(day,i) in days"><br v-if="i && i % 7===0"/><span class="day" :class="{empty: !day || !day.snapshotCount, 'contains-current-snapshot':dayContainsCurrentSnapshot(day)}" :style="dayStyle"  @click="gotoDay(day, $event)" @keyup.13="gotoDay(day, $event)"><template v-if="day"><span class="size" v-if="day.snapshotCount" :style="getDayCountCircleStyle(day.snapshotCount)" tabindex="0"> </span><span class="day-id" :class="{empty: !day.snapshotCount}">{{day.id}}</span><span v-if="day.snapshotCount" class="count">{{ $root._(day.snapshotCount !== 1 ? '{count} captures':'{count} capture', {count: day.snapshotCount}) }}</span></template><template v-else v-html="'&nbsp;'"></template></span></span>
+  </div>
+</template>
+
+<script>
+import {PywbI18N} from "../i18n";
+
+export default {
+  props: ["month", "year", "isCurrent", "yearContainsCurrentSnapshot", "currentSnapshot"],
+  data: function() {
+    return {
+      maxInDay: 0,
+      daySize: 30,
+    };
+  },
+  computed: {
+    dayInitials() {
+      return PywbI18N.instance.getWeekDays().map(d => d.substr(0,1));
+    },
+    dayStyle() {
+      const s = this.daySize;
+      return `height: ${s}px; width: ${s}px; line-height: ${s}px`;
+    },
+    days() {
+      if (!this.month) {
+        return [];
+      }
+      const days = [];
+      // Get days in month, and days in the complete weeks before first day and after last day
+      const [firstDay, lastDay] = this.month.getChildrenRange();
+      const daysBeforeFirst = (7 + (new Date(this.year.id, this.month.id-1, firstDay)).getDay() - PywbI18N.firstDayOfWeek) % 7;
+      const daysAfterLastDay = (6 - (new Date(this.year.id, this.month.id-1, lastDay)).getDay() + PywbI18N.firstDayOfWeek) % 7;
+      for(let i=0; i<daysBeforeFirst; i++) {
+        days.push(null);
+      }
+      const hasChildren = !!this.month.children.length;
+      for(let i=0; i<lastDay; i++) {
+        days.push(hasChildren ? this.month.children[i] : null);
+      }
+      for(let i=0; i<daysAfterLastDay; i++) {
+        days.push(null);
+      }
+      return days;
+    },
+    containsCurrentSnapshot() {
+      return this.currentSnapshot &&
+          this.month.contains(this.currentSnapshot);
+    }
+  },
+  methods: {
+    _(id, embeddedVariableStrings=null) {
+      return PywbI18N.instance.getText(id, embeddedVariableStrings);
+    },
+    getLongMonthName(id) {
+      return PywbI18N.instance.getMonth(id);
+    },
+    gotoDay(day, event) {
+      if (!day || !day.snapshotCount) {
+        return;
+      }
+      // upon doing to day, tell timeline to highlight itself
+      // this.$root.timelineHighlight = true;
+      this.$emit("show-day-timeline", day, event);
+    },
+    getDayCountCircleStyle(snapshotCount) {
+      const size = Math.ceil((snapshotCount/this.year.maxGrandchildSnapshotCount) * this.daySize);
+      const scaledSize = size ? (this.daySize*.3 + Math.ceil(size*.7)) : 0;
+      const margin = (this.daySize-scaledSize)/2;
+
+      // TEMPORARILY DISABLE AUTO-HUE calculation as it is contributing to better understand of data
+      // color hue should go form blue (240deg) to red (360deg)
+      // const colorHue = Math.ceil((snapshotCount/this.year.maxGrandchildSnapshotCount) * (360-240));
+      // const scaledColorHue = size ? (240 + colorHue) : 240;
+      // background-color: hsl(${scaledColorHue}, 100%, 50%, .2)
+
+      return `width: ${scaledSize}px; height: ${scaledSize}px; top: ${margin}px; left: ${margin}px; border-radius: ${scaledSize/2}px;`;
+    },
+    dayContainsCurrentSnapshot(day) {
+      return !!day && day.snapshotCount > 0 && this.containsCurrentSnapshot && day.contains(this.currentSnapshot);
+    }
+  }
+};
+</script>
+

pywb/vueui/src/components/CalendarYear.vue

@@ -0,0 +1,186 @@
+<template>
+  <div class="full-view border-top-0 border-left-0 border-right-0 border-bottom border-dark shadow">
+    <h2>
+      <i
+        class="fas fa-arrow-left year-arrow"
+        @click="gotoPreviousYear"
+        @keyup.enter="gotoPreviousYear"
+        v-if="previousYear"
+        tabindex="0"></i>
+      <span class="mx-1">
+      {{year.id}} ({{ $root._(year.snapshotCount !== 1 ? '{count} captures':'{count} capture', {count: year.snapshotCount}) }})
+      </span>
+      <i
+        class="fas fa-arrow-right year-arrow"
+        @click="gotoNextYear"
+        @keyup.enter="gotoNextYear"
+        v-if="nextYear"
+        tabindex="0"></i>
+    </h2>
+    <div class="months">
+      <CalendarMonth
+        v-for="month in year.children"
+        :key="month.id"
+        :month="month"
+        :year="year"
+        :current-snapshot="containsCurrentSnapshot ? currentSnapshot : null"
+        :is-current="month === currentMonth"
+        @goto-period="$emit('goto-period', $event)"
+        @show-day-timeline="setCurrentTimeline"
+      ></CalendarMonth>
+    </div>
+    <Tooltip
+      :position="currentTimelinePos"
+      v-if="currentTimelinePeriod"
+      ref="timelineLinearTooltip">
+      <TimelineLinear
+        :period="currentTimelinePeriod"
+        :current-snapshot="containsCurrentSnapshot ? currentSnapshot : null"
+        @goto-period="gotoPeriod"
+      ></TimelineLinear>
+    </Tooltip>
+  </div>
+</template>
+
+<script>
+import CalendarMonth from "./CalendarMonth.vue";
+import TimelineLinear from "./TimelineLinear.vue";
+import Tooltip from "./Tooltip.vue";
+import { PywbPeriod } from "../model.js";
+
+export default {
+  components: {CalendarMonth, TimelineLinear, Tooltip},
+  props: ["period", "currentSnapshot"],
+  data: function() {
+    return {
+      firstZoomLevel: PywbPeriod.Type.day,
+      currentTimelinePeriod: null,
+      currentTimelinePos: '0,0'
+    };
+  },
+  mounted() {
+    document.querySelector('body').addEventListener('click', this.resetCurrentTimeline);
+  },
+  computed: {
+    year() { // the year that the timeline period is in
+      let year = null;
+      // if timeline is showing all year
+      if (this.period.type === PywbPeriod.Type.all) {
+        // if no current snapshot => pick the LAST YEAR
+        if (!this.currentSnapshot) {
+          year = this.period.children[this.period.children.length-1];
+        } else {
+          year = this.period.findByFullId(String(this.currentSnapshot.year));
+        }
+      } else if (this.period.type === PywbPeriod.Type.year) {
+        year = this.period;
+      } else {
+        year = this.period.getParents().filter(p => p.type === PywbPeriod.Type.year)[0];
+      }
+      if (year) {
+        year.fillEmptyChildPeriods(true);
+      }
+      return year;
+    },
+    currentYearIndex() {
+      if (this.year.parent) {
+        return this.year.parent.children.findIndex(year => year.fullId === this.year.fullId);
+      }
+    },
+    previousYear() {
+      return this.year.getPrevious();
+    },
+    nextYear() {
+      return this.year.getNext();
+    },
+    currentMonth() { // the month that the timeline period is in
+      let month = null;
+      if (this.period.type === PywbPeriod.Type.month) {
+        month = this.period;
+      } else {
+        month = this.period.getParents().filter(p => p.type === PywbPeriod.Type.month)[0];
+      }
+      return month;
+    },
+    containsCurrentSnapshot() {
+      return this.currentSnapshot &&
+        this.year.contains(this.currentSnapshot);
+    }
+  },
+  methods: {
+    gotoPreviousYear() {
+      this.gotoPeriod(this.previousYear, true /* changeYearOnly */);
+    },
+    gotoNextYear() {
+      this.gotoPeriod(this.nextYear, true /* changeYearOnly */);
+    },
+    resetCurrentTimeline(event) {
+      if (event && this.$refs.timelineLinearTooltip) {
+        let el = event.target;
+        let clickWithinTooltip = false;
+        while(el.parentElement) {
+          if (el === this.$refs.timelineLinearTooltip.$el) {
+            clickWithinTooltip = true;
+            break;
+          }
+          el = el.parentElement;
+        }
+        if (!clickWithinTooltip) {
+          this.currentTimelinePeriod = null;
+        }
+      }
+    },
+    setCurrentTimeline(day, event) {
+      this.currentTimelinePeriod = day;
+      if (!day) {
+        return;
+      }
+      if (event.code === "Enter") {
+        let middleXPos = (window.innerWidth / 2) - 60;
+        this.currentTimelinePos = `${middleXPos},200`;
+      } else {
+        this.currentTimelinePos = `${event.x},${event.y}`;
+      }
+      
+      event.stopPropagation();
+      event.preventDefault();
+    },
+    gotoPeriod(period, changeYearOnly=false) {
+      if (period.snapshot || period.snapshotPeriod || changeYearOnly) {
+        this.$emit('goto-period', period);
+      } else {
+        this.currentTimelinePeriod = period;
+      }
+    },
+
+  }
+};
+</script>
+
+<style scoped>
+  .full-view {
+    display: flex;
+    flex: 1;
+    flex-wrap: wrap;
+    z-index: 10;
+    overflow-y: auto;
+    width: 100%;
+    background-color: white;
+    padding-bottom: 1em;
+    justify-content: center;
+  }
+  .full-view .months {
+    display: flex;
+    justify-content: center;
+    flex-wrap: wrap;
+    align-items: flex-start;
+  }
+  .full-view h2 {
+    margin: 10px 0;
+    font-size: 20px;
+    text-align: center;
+  }
+  .year-arrow:hover {
+    cursor: pointer;
+  }
+</style>

pywb/vueui/src/components/LoadingSpinner.vue

@@ -0,0 +1,139 @@
+<template>
+  <div class="pywb-loading-spinner-mask" :class="{hidden: isHidden, spinning: isSpinning}" @click="setOff">
+    <div class="pywb-loading-spinner">
+      <div data-loading-spinner="circle1"></div>
+      <div data-loading-spinner="circle2"></div>
+      <div data-loading-spinner="circle3"></div>
+      <div data-loading-spinner="circle4"></div>
+      <span data-loading-spinner="text">{{text}}</span>
+    </div>
+  </div>
+</template>
+
+<script>
+let elStyle = null;
+export default {
+  name: "LoadingSpinner",
+  props: ['text', 'isLoading'],
+  mounted() {
+    this.setOnOff();
+    this.$watch('isLoading', this.setOnOff);
+  },
+  data() {
+    return {
+      isSpinning: '',
+      isHidden: ''
+    }
+  },
+  methods: {
+    setOnOff() {
+      if (this.isLoading) {
+        this.setOn();
+      } else {
+        this.setOff();
+      }
+    },
+    setOn() {
+      this.isHidden = false;
+      setTimeout(function setSpinning() {
+        this.isSpinning = true;
+      }.bind(this), 100);
+    },
+    setOff() {
+      this.isSpinning = false;
+      setTimeout(function setHidden() {
+        this.isHidden = true;
+      }.bind(this), 500);
+    }
+  }
+}
+</script>
+
+<style scoped>
+.pywb-loading-spinner-mask.spinning {
+  opacity: 1;
+}
+.pywb-loading-spinner-mask.hidden {
+  display: none;
+}
+.pywb-loading-spinner-mask {
+  position: fixed;
+  top: 10px;
+  left: 0;
+  width: 100vw;
+  height: 100vh;
+  z-index: 900;
+
+  display: flex;
+  justify-content: center;
+  align-items: center;
+
+  background-color: rgba(255,255,255, .85);
+
+  opacity: 0;
+  transition: opacity 500ms ease-in;
+}
+
+.pywb-loading-spinner {
+  position: relative;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  width: 200px;
+  height: 200px;
+}
+[data-loading-spinner^=circle] {
+  position: absolute;
+  border: 3px solid #444444;/* #0D4B9F; */
+  width: 70%;
+  height: 70%;
+  margin: 0;
+  border-radius: 50%;
+  border-left-color: transparent;
+  border-right-color: transparent;
+  animation: rotate 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}
+[data-loading-spinner=circle2] {
+  border: 3px solid #ddd;/* #E0EDFF; */
+  width: 80%;
+  height: 80%;
+  border-left-color: transparent;
+  border-right-color: transparent;
+  animation: rotate2 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}
+[data-loading-spinner=circle3] {
+  border: 3px solid #656565;/* #005CDC; */
+  width: 90%;
+  height: 90%;
+  border-left-color: transparent;
+  border-right-color: transparent;
+  animation: rotate 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}
+[data-loading-spinner=circle4] {
+  border: 3px solid #aaa; /* #94B6E5; */
+  width: 100%;
+  height: 100%;
+  border-left-color: transparent;
+  border-right-color: transparent;
+  animation: rotate2 2s cubic-bezier(0.26, 1.36, 0.74, -0.29) infinite;
+}
+@keyframes rotate {
+  0% {
+    transform: rotateZ(-360deg)
+  }
+  100% {
+    transform: rotateZ(0deg)
+  }
+}
+@keyframes rotate2 {
+  0% {
+    transform: rotateZ(360deg)
+  }
+  100% {
+    transform: rotateZ(0deg)
+  }
+}
+[data-loading-spinner=text] {
+  font-size: 15px;
+}
+</style>

pywb/vueui/src/components/PageRuler.vue

@@ -0,0 +1,151 @@
+<template>
+    <div class="ruler">
+      <div class="inner" :class="{hidden: !isShow}">
+        <div class="axis x" :style="{width: xAxisWidth}" ref="xAxis">
+          <div class="tick x" v-for="x in xTicks" >{{ x ? x : '' }}</div>
+          <div class="line x" v-if="xAxisLine" :style="{left: xAxisLine+'px'}"><span>{{xAxisLine}}</span></div>
+        </div>
+        <div class="axis y" :style="{height: yAxisHeight}" ref="yAxis">
+          <div class="tick y" v-for="y in yTicks">{{ y ? y : '' }}</div>
+          <div class="line y" v-if="yAxisLine" :style="{top: yAxisLine+'px'}"><span>{{yAxisLine}}</span></div>
+        </div>
+      </div>
+        <span class="toggle" @click="isShow=!isShow">R</span>
+    </div>
+</template>
+
+<style>
+.ruler {
+  position: fixed;
+  margin: 0;
+  padding: 0;
+  z-index: 90;
+}
+.ruler .inner.hidden { display: none; }
+
+  .ruler .toggle {
+    cursor: pointer;
+    z-index: 91;
+    position: fixed;
+    top: 0;
+    left: 0;
+  }
+  .ruler .line {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 10px;
+    height: 10px;
+  }
+  .ruler .line  span {
+    color: red;
+    font-weight: bold;
+    background-color: rgba(255,255,255,.9);
+  }
+
+  .ruler .tick {
+    font-size: 15px; line-height: 1;
+  }
+
+  .ruler .axis {
+    background-color: rgba(255,255,255,.9);
+    position: fixed;
+    top: 0;
+    left: 0;
+    white-space: nowrap;
+    overflow: visible;
+  }
+
+  .ruler .axis.x { height: 15px;}
+  .ruler .tick.x {
+    display: inline-block;
+    height: 15px;
+    width: 49px;
+    border-left: 1px solid black;
+  }
+  .ruler .axis.y { width: 15px; }
+  .ruler .tick.y {
+    display: block;
+    width: 15px;
+    height: 49px;
+    border-top: 1px solid black;
+  }
+  .ruler .line.x {
+    height: 100vh;
+    border-left: 1px solid red;
+  }
+  .ruler .line.y {
+    width: 100vw;
+    border-top: 1px solid red;
+  }
+</style>
+
+<script>
+export default {
+  data() {
+    return {
+      isShow: false,
+      xTicks: [],
+      xAxisLine: 0,
+      yTicks: [],
+      yAxisLine: 0,
+      increment: 50 //px
+    }
+  },
+  mounted() {
+    for (let i = 0; i < 100; i++) {
+      this.xTicks.push(i * this.increment);
+      this.yTicks.push(i * this.increment);
+    }
+
+    const onMouseMoveHandlerFn_ = (event) => {
+      if (event.currentTarget === this.$refs.xAxis) {
+        this.xAxisLine = event.x;
+        console.log(event.x);
+      } else if (event.currentTarget === this.$refs.yAxis) {
+        this.yAxisLine = event.y;
+        console.log(event.y);
+      }
+    };
+    const onMouseMoveHandlerFn = onMouseMoveHandlerFn_.bind(this);
+    const toggleHandlers = (event) => {
+      if (event.currentTarget === this.$refs.xAxis) {
+        if (this.$refs.xAxis.isGuidelineOn) {
+          this.$refs.xAxis.isGuidelineOn = false;
+          this.xAxisLine = 0;
+          this.$refs.xAxis.removeEventListener('mousemove', onMouseMoveHandlerFn, true);
+        } else {
+          this.$refs.xAxis.isGuidelineOn = true;
+          this.xAxisLine = event.x;
+          this.$refs.xAxis.addEventListener('mousemove', onMouseMoveHandlerFn, true);
+        }
+        event.stopPropagation();
+      } else if (event.currentTarget === this.$refs.yAxis) {
+        if (this.$refs.yAxis.isGuidelineOn) {
+          this.$refs.yAxis.isGuidelineOn = false;
+          this.yAxisLine = 0;
+          this.$refs.yAxis.removeEventListener('mousemove', onMouseMoveHandlerFn, true);
+        } else {
+          this.$refs.yAxis.isGuidelineOn = true;
+          this.yAxisLine = event.y;
+          this.$refs.yAxis.addEventListener('mousemove', onMouseMoveHandlerFn, true);
+        }
+        event.stopPropagation();
+      }
+    }
+
+    this.$refs.xAxis.addEventListener('click', toggleHandlers.bind(this), true);
+    this.$refs.yAxis.addEventListener('click', toggleHandlers.bind(this), true);
+  },
+  computed: {
+    yAxisHeight() {
+      return `${this.yTicks.length * this.increment}px`;
+    },
+    xAxisWidth() {
+      return `${this.xTicks.length * this.increment}px`;
+    }
+  },
+  methods: {
+  }
+};
+</script>

pywb/vueui/src/components/Timeline.vue

@@ -0,0 +1,446 @@
+<template>
+    <div class="timeline">
+        <div class="period-tooltip" v-show="tooltipPeriod" :style="{left: tooltipPeriodPos.x+'px', top: tooltipPeriodPos.y+'px'}">
+            <template v-if="tooltipPeriod">
+              <div v-if="tooltipPeriod.snapshot">
+                {{ $root._('View capture on {date}', {date: tooltipPeriod.snapshot.getTimeDateFormatted()}) }}
+              </div>
+              <div v-else-if="tooltipPeriod.snapshotPeriod">
+                {{ $root._('View capture on {date}', {date: tooltipPeriod.snapshotPeriod.snapshot.getTimeDateFormatted()}) }}
+              </div>
+              <div v-else-if="tooltipPeriod.snapshotCount">
+                {{ $root._(
+                  isTooltipPeriodDayOrHour ? '{capture_text} on {date}':'{capture_text} in {month}', // TODO: split translation into "in {year}" and "in {month}"
+                  { capture_text: $root._(tooltipPeriod.snapshotCount !== 1 ? '{count} captures' : '{count} capture', {count: tooltipPeriod.snapshotCount}), [isTooltipPeriodDayOrHour ? 'date':'month']: tooltipPeriod.getFullReadableId() } )
+                }}
+              </div>
+            </template>
+        </div>
+        <div v-html="'&#x25C0;'"
+          class="arrow previous"
+          :class="{disabled: isScrollZero && !previousPeriod}"
+          @click="scrollPrev"
+          @keyup.enter="scrollPrev"
+          @dblclick.stop.prevent tabindex="0"></div>
+        <div class="scroll" ref="periodScroll" :class="{highlight: highlight}">
+            <div class="periods" ref="periods">
+                <div v-for="subPeriod in period.children"
+                     :key="subPeriod.fullId"
+                     class="period"
+                     :class="{empty: !subPeriod.snapshotCount, highlight: highlightPeriod === subPeriod, 'last-level': !canZoom, 'contains-current-snapshot': containsCurrentSnapshot(subPeriod) }"
+                >
+                    <div class="histo">
+                        <div class="line"
+                             v-for="histoPeriod in subPeriod.children"
+                             :key="histoPeriod.fullId"
+                             :style="{height: getHistoLineHeight(histoPeriod.snapshotCount)}"
+                             :class="{'has-single-snapshot': histoPeriod.snapshotCount === 1, 'contains-current-snapshot': containsCurrentSnapshot(histoPeriod)}"
+                             @click="changePeriod(histoPeriod, $event)"
+                             @keyup.enter="changePeriod(histoPeriod, $event)"
+                             @mouseover="setTooltipPeriod(histoPeriod, $event)"
+                             @mouseout="setTooltipPeriod(null, $event)"
+                             :tabindex="histoPeriod.snapshotCount > 0 ? 0 : -1"
+                        >
+                        </div>
+                    </div>
+                    <div class="inner"
+                         :class="{'has-single-snapshot': subPeriod.snapshotCount === 1}"
+                         @click="changePeriod(subPeriod, $event)"
+                         @keyup.enter="changePeriod(histoPeriod, $event)"
+                         @mouseover="setTooltipPeriod(subPeriod, $event)"
+                         @mouseout="setTooltipPeriod(null, $event)"
+                    >
+                        <div class="label">
+                          {{subPeriod.getReadableId()}}
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div
+          v-html="'&#x25B6;'"
+          class="arrow next"
+          :class="{disabled: isScrollMax && !nextPeriod}"
+          @click="scrollNext"
+          @keyup.enter="scrollNext"
+          @dblclick.stop.prevent tabindex="0"></div>
+    </div>
+</template>
+
+<script>
+import { PywbPeriod } from "../model.js";
+
+export default{
+  props: {
+    period: { required: true },
+    currentSnapshot: { required: false, default: null},
+    highlight: { required: false, default: false},
+    stayWithinPeriod: { required: false, default: false},
+    maxZoomLevel: { required: false, default: PywbPeriod.Type.snapshot}
+  },
+  data: function() {
+    return {
+      highlightPeriod: null,
+      previousPeriod: null,
+      nextPeriod: null,
+      isScrollZero: true,
+      isScrollMax: true,
+      tooltipPeriod: null,
+      tooltipPeriodPos: {x:0,y:0}
+    };
+  },
+  created: function() {
+    this.addEmptySubPeriods();
+  },
+  mounted: function() {
+    this.$refs.periods._computedStyle = window.getComputedStyle(this.$refs.periods);
+    this.$refs.periodScroll._computedStyle = window.getComputedStyle(this.$refs.periodScroll);
+    this.$watch("period", this.onPeriodChanged);
+
+    this.$refs.periodScroll.addEventListener("scroll", this.updateScrollArrows);
+    window.addEventListener("resize", this.updateScrollArrows);
+    this.updateScrollArrows();
+  },
+  computed: {
+    // this determins which the last zoom level is before we go straight to showing snapshot
+    canZoom() {
+      return this.period.type < this.maxZoomLevel;
+    },
+    isTooltipPeriodDayOrHour() {
+      return this.tooltipPeriod.type >= PywbPeriod.Type.day;
+    },
+    iContainCurrentSnapshot() {
+      return this.currentSnapshot && this.period.contains(this.currentSnapshot);
+    }
+  },
+  updated() {
+    // do something on update
+  },
+  methods: {
+    containsCurrentSnapshot(period) {
+      return this.iContainCurrentSnapshot && period.contains(this.currentSnapshot);
+    },
+    addEmptySubPeriods() {
+      this.period.fillEmptyChildPeriods(true);
+    },
+    updateScrollArrows() {
+      this.period.scroll = this.$refs.periodScroll.scrollLeft;
+      const maxScroll = parseInt(this.$refs.periods._computedStyle.width) - parseInt(this.$refs.periodScroll._computedStyle.width);
+      this.isScrollZero = !this.period.scroll; // if 0, then true (we are at scroll zero)
+      this.isScrollMax = Math.abs(maxScroll - this.period.scroll) < 5;
+    },
+    restoreScroll() {
+      this.$refs.periodScroll.scrollLeft = this.period.scroll;
+    },
+    scrollNext: function () {
+      if (this.isScrollMax) {
+        if (this.nextPeriod) {
+          this.$emit("goto-period", this.nextPeriod, true /* onlyZoomToPeriod */);
+        }
+      } else {
+        this.$refs.periodScroll.scrollLeft += 30;
+      }
+    },
+    scrollPrev: function () {
+      if (this.isScrollZero) {
+        if (this.previousPeriod) {
+          this.$emit("goto-period", this.previousPeriod, true /* onlyZoomToPeriod */);
+        }
+      } else {
+        this.$refs.periodScroll.scrollLeft -= 30;
+      }
+    },
+    getTimeFormatted: function(date) {
+      return (date.hour < 13 ? date.hour : (date.hour % 12)) + ":" + ((date.minute < 10 ? "0":"")+date.minute) + " " + (date.hour < 12 ? "am":"pm");
+    },
+    getHistoLineHeight: function(value) {
+      const percent = Math.ceil((value/this.period.maxGrandchildSnapshotCount) * 100);
+      return (percent ? (5 + Math.ceil(percent*.95)) : 0) + "%";
+      // return percent + '%';
+    },
+    changePeriod(period, $event) {
+      // if not empty
+      if (period.snapshotCount) {
+        let periodToChangeTo = null;
+        // if contains a single snapshot only, navigate to snapshot (load snapshot in FRAME, do not ZOOM IN)
+        if (period.snapshot) {
+          // if period is at level "snapshot" (no more children), send period, else send the child period, a reference to which is stored (by data/model layer) in the current period; App event needs a period to be passed (cannot pass in snapshot object itself)
+          if (period.type === PywbPeriod.Type.snapshot) {
+            periodToChangeTo = period;
+          } else if (period.snapshotPeriod) {
+            periodToChangeTo = period.snapshotPeriod;
+          }
+        } else {
+        // if contains mulitple snapshots,
+          // zoom if ZOOM level is day or less, OR if period contain TOO MANY (>10)
+          if (this.canZoom) {
+            periodToChangeTo = period;
+          }
+        }
+
+        // if we selected a period to go to, emit event
+        if (periodToChangeTo) {
+          this.$emit("goto-period", periodToChangeTo);
+        }
+      }
+      $event.stopPropagation();
+      return false;
+    },
+    onPeriodChanged(newPeriod, oldPeriod) {
+      this.addEmptySubPeriods();
+      const previousPeriod = this.period.getPrevious();
+      const nextPeriod = this.period.getNext();
+      if (!this.stayWithinPeriod || this.stayWithinPeriod.contains(previousPeriod)) {
+        this.previousPeriod = previousPeriod;
+      }
+      if (!this.stayWithinPeriod || this.stayWithinPeriod.contains(nextPeriod)) {
+        this.nextPeriod = nextPeriod;
+      }
+
+      // detect if going up level of period (new period type should be in old period parents)
+      if (oldPeriod && oldPeriod.type - newPeriod.type > 0) {
+        let highlightPeriod = oldPeriod;
+        for (let i=oldPeriod.type - newPeriod.type; i > 1; i--) {
+          highlightPeriod = highlightPeriod.parent;
+        }
+        this.highlightPeriod = highlightPeriod;
+        setTimeout((function() {
+          this.highlightPeriod = null;
+        }).bind(this), 2000);
+      }
+      setTimeout((function() {
+        this.restoreScroll();
+        this.updateScrollArrows();
+      }).bind(this), 1);
+    },
+    setTooltipPeriod(period, event) {
+      if (!period || !period.snapshotCount) {
+        this.tooltipPeriod = null;
+        return;
+      }
+      this.tooltipPeriod = period;
+
+      this.$nextTick(function() {
+        const tooltipContentsEl = document.querySelector('.period-tooltip div');
+        if (!tooltipContentsEl) {
+          return;
+        }
+
+        const periodTooltipStyle = window.getComputedStyle(tooltipContentsEl);
+        const tooltipWidth = parseInt(periodTooltipStyle.width);
+        const tooltipHeight = parseInt(periodTooltipStyle.height);
+        const spacing = 10;
+        if (window.innerWidth < event.x + (spacing*2) + tooltipWidth) {
+          this.tooltipPeriodPos.x = event.x - (tooltipWidth + spacing);
+        } else {
+          this.tooltipPeriodPos.x = event.x + spacing;
+        }
+        this.tooltipPeriodPos.y = event.y - (spacing + tooltipHeight);
+      });
+      event.stopPropagation();
+      return false;
+    }
+  }
+};
+</script>
+
+
+<style>
+    .timeline {
+        position: relative;
+        display: flex;
+        width: auto;
+        height: 60px;
+        margin: 5px;
+        justify-content: left;
+    }
+
+    .timeline .id {
+        display: inline-block;
+        font-size: 30px;
+    }
+    .timeline .arrow {
+        display: inline-block;
+        width: 20px;
+        font-size: 20px; /* font-size = width of arrow, as it UTF char */
+        line-height: 60px;
+        vertical-align: top;
+        cursor: pointer;
+    }
+    .timeline .arrow.previous {
+    }
+    .timeline .arrow.next {
+    }
+    .timeline .arrow.disabled, .timeline .arrow.disabled:hover {
+        /*color: lightgray;*/
+        background-color: transparent;
+        /*cursor: not-allowed;*/
+        visibility: hidden;
+    }
+    .timeline .arrow:hover {
+        background-color: antiquewhite;
+        color: firebrick;
+    }
+
+    .timeline .scroll {
+        position: relative;
+        display: inline-block;
+        width: 100%; /* */
+        height: 100%;
+
+        /* maker scrollable horizontally */
+        overflow-x: scroll;
+        overflow-y: hidden;
+        white-space: nowrap;
+        scroll-behavior: smooth;
+
+        text-align: center;
+
+        transition: background-color 500ms ease-in;
+    }
+    /* hide scroll bar */
+    .timeline .scroll::-webkit-scrollbar {
+        display: none;
+    }
+    /* highlight the scroll period: usually triggered from root app */
+    .timeline .scroll.highlight {
+        background-color: #fff7ce;
+    }
+    .timeline .scroll .periods {
+        display: flex;
+        justify-content: space-between;
+        height: 100%;
+        width: 100%;
+        min-width: 600px;
+    }
+
+
+    .timeline .period {
+        flex-grow: 1;
+        position: relative;
+        display: inline-block;
+        height: 100%;
+        /* line-height: 80px; /* use to center middle vertically */
+        white-space: normal;
+        vertical-align: top;
+        text-align: center;
+        background-color: transparent;
+
+        transition: background-color 500ms ease-in-out;
+    }
+    /* 1st period period child el */
+    .timeline .period:nth-child(1) {
+        /*border-left: 1px solid white; !* has left border; all other periods have right border *!*/
+    }
+
+    .timeline .period:hover {
+        background-color: #eeeeee;
+    }
+    .timeline .period.contains-current-snapshot, .timeline .period.contains-current-snapshot:hover {
+        background-color: #f7def4;
+    }
+
+    /* empty period */
+    .timeline .period.empty {
+        color: #aaa;
+        /*background-color: transparent;*/
+    }
+    /* highlighted period */
+    .timeline .period.highlight {
+        background-color: cyan;
+    }
+
+    .timeline .period .inner {
+        display: block;
+        position: absolute;
+        bottom: 0;
+        left: 0;
+        width: 100%;
+        height: 20px;
+        background-color: white;
+        border-top: 1px solid gray;
+        white-space: nowrap;
+        cursor: zoom-in;
+    }
+    .timeline .period .inner.has-single-snapshot {
+      cursor: pointer;
+    }
+    .timeline .period.last-level .inner, .timeline .period.empty .inner {
+      cursor: default;
+    }
+
+    .timeline .period .label {
+      width: 100%;
+      font-weight: bold;
+      font-size: 14px;
+      transition: background-color 500ms ease-in;
+    }
+    .timeline .period:hover .label {
+      position: absolute;
+      z-index: 20;
+      background-color: lightgrey;
+    }
+
+    .timeline .period .histo {
+        display: flex;
+        position: absolute;
+        top: 1px;
+        left: 0;
+        width: 100%;
+        height: 39px;
+        align-items: flex-end;
+        justify-content: space-between;
+        text-align: left;
+    }
+
+    .timeline .period .histo .line {
+        position: relative;
+        flex-grow: 1;
+        display: inline-block;
+        background-color: #a6cdf5;
+        margin: 0;
+        padding: 0;
+        cursor: zoom-in;
+    }
+    .timeline .period .histo .line.has-single-snapshot {
+      cursor: pointer;
+    }
+
+    /* Last level period histogram spaces things evenly */
+    .timeline .period.last-level .histo {
+        justify-content: space-around;
+    }
+
+    /* Last level period histogram lines do not grow, but are fixed width/margin */
+    .timeline .period.last-level .histo .line {
+        flex-grow: unset;
+        width: 5px;
+        margin-left: 2px;
+    }
+
+        /* update line color on hover*/
+        .timeline .period .histo .line:hover {
+            background-color: #f5a6eb;
+        }
+
+        .timeline .period .histo .line.contains-current-snapshot {
+            background-color: red;
+        }
+
+        /* Period that contains ONE snapshot only will show snapshot info*/
+        .timeline .period-tooltip {
+            position: fixed;
+            z-index: 100;
+            /*left or right set programmatically*/
+            display: block;
+            background-color: white;
+            border: 1px solid gray;
+            padding: 2px;
+            white-space: nowrap; /*no wrapping allowed*/
+        }
+            /*show on hover*/
+            .timeline .period-tooltip.show {
+                display: block;
+            }
+
+</style>

pywb/vueui/src/components/TimelineBreadcrumbs.vue

@@ -0,0 +1,92 @@
+<template>
+    <div class="breadcrumbs">
+        <template v-if="parents.length">
+            <span class="item">
+                <span
+                    class="goto"
+                    @click="changePeriod(parents[0])"
+                    @keyup.enter="changePeriod(parents[0])"
+                    :title="getPeriodZoomOutText(parents[0])"
+                    tabindex="1">
+                  <i class="fa fa-search-minus"></i> {{parents[0].getReadableId(true)}}
+                </span>
+            </span>
+            &gt;
+            <span v-for="(parent,i) in parents" :key="parent.id" class="item" v-if="i > 0">
+                <span
+                    class="goto"
+                    @click="changePeriod(parent)"
+                    @keyup.enter="changePeriod(parent)"
+                    :title="getPeriodZoomOutText(parent)"
+                    tabindex="1">
+                  {{parent.getReadableId(true)}}
+                </span>
+            </span>
+        </template>
+        <span class="item">
+            <span class="current">{{period.getReadableId(true)}}</span>
+            <span class="count">({{ $root._(period.snapshotCount !== 1 ? '{count} captures':'{count} capture', {count: period.snapshotCount}) }})</span>
+        </span>
+    </div>
+</template>
+
+<script>
+export default {
+  props: {
+    period: {
+      required: true
+    }
+  },
+  computed: {
+    parents: function() {
+      return this.period.getParents();
+    }
+  },
+  methods: {
+    getPeriodZoomOutText(period) {
+      return 'Zoom out to '+period.getReadableId(true)+ ' ('+period.snapshotCount+' captures)';
+    },
+    changePeriod(period) {
+      if (period.snapshotCount) {
+        this.$emit("goto-period", period);
+      }
+    },
+  }
+};
+</script>
+
+<style>
+    .breadcrumbs {
+      text-align: center;
+    }
+    .breadcrumbs .item {
+        position: relative;
+        display: inline;
+        margin: 0 2px 0 0;
+        font-size: inherit;
+    }
+    .breadcrumbs .count {
+        /*vertical-align: middle;*/
+        font-size: inherit;
+    }
+
+    .breadcrumbs .item .goto {
+        display: inline-block;
+        margin: 1px;
+        padding: 1px;
+        cursor: zoom-out;
+        border-radius: 5px;
+        background-color: #eeeeee;
+     }
+    .breadcrumbs .item .goto:hover {
+        background-color: #a6cdf5;
+    }
+    .breadcrumbs .item .goto img {
+      height: 15px;
+    }
+
+    .breadcrumbs .item.snapshot {
+        display: block;
+    }
+
+</style>

pywb/vueui/src/components/TimelineLinear.vue

@@ -0,0 +1,90 @@
+<template>
+  <div class="timeline-linear">
+    <div class="title">
+      <div>{{ displayDate }}</div>
+      <div>{{ $root._(period.snapshotCount !== 1 ? '{count} captures':'{count} capture', {count: period.snapshotCount}) }}</div>
+    </div>
+
+    <div class="list">
+      <div v-for="snapshotPeriod in snapshotPeriods">
+        <span
+          @click="changePeriod(snapshotPeriod)"
+          @keyup.enter="changePeriod(snapshotPeriod)"
+          class="link"
+          tabindex="1">
+          {{ snapshotPeriod.snapshot.getTimeFormatted() }}
+        </span>
+        <span v-if="isCurrentSnapshot(period)" class="current">{{$root._('current')}}</span>
+      </div>
+    </div>
+  </div>
+</template>
+
+<script>
+import { PywbI18N } from "../i18n";
+
+export default {
+  name: "TimelineLinear",
+  props: ['period', 'currentSnapshot'],
+  computed: {
+    snapshotPeriods() {
+      return this.period.getSnapshotPeriodsFlat();
+    },
+    containsCurrentSnapshot() {
+      return this.currentSnapshot &&
+          this.period.contains(this.currentSnapshot);
+    },
+    displayDate() {
+      // replace '-' in date string with '/' so firefox and chrome will return same result
+      let dateStringNoSlashes = this.period.fullId.replace(/-/g,'/');
+      return new Date(dateStringNoSlashes).toLocaleDateString(PywbI18N.getLocale());
+    }
+  },
+  methods: {
+    isCurrentSnapshot(period) {
+      if (!!this.currentSnapshot && !!period.snapshot) {
+        return this.currentSnapshot && this.currentSnapshot.id === period.snapshot.id;
+      }
+      return false;
+    },
+    changePeriod(period) {
+      this.$emit("goto-period", period);
+    }
+  }
+}
+</script>
+
+<style scoped>
+.timeline-linear {
+  width: auto;
+  padding: 5px;
+  background-color: white;
+  border: 1px solid gray;
+  border-radius: 5px;
+  z-index: 1100;
+}
+.timeline-linear .list {
+  max-height: 80vh;
+  min-height: 50px;
+  overflow-x: hidden;
+  overflow-y: auto;
+}
+.timeline-linear .title {
+  border-bottom: 1px solid black;
+  font-weight: bold;
+  font-family: Arial, sans-serif;
+}
+.timeline-linear .link {
+  text-decoration: underline;
+  color: darkblue;
+}
+.timeline-linear .link:hover {
+  color: lightseagreen;
+  cursor: pointer;
+}
+.timeline-linear .current {
+  background-color: deeppink;
+  color: white;
+  border-radius: 5px;
+}
+</style>