diff --git a/.gitignore b/.gitignore index 020f9704..cd0bc5c7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ develop-eggs lib lib64 __pycache__ +.DS_Store # ignore auto-gen certs ca/pywb-ca.pem @@ -47,3 +48,6 @@ node_modules/ # git_hash git_hash.py + +# Sphinx documentation +docs/_build/* diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 01f56e3a..00000000 --- a/doc/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pywb.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pywb.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/pywb" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pywb" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/conf.py b/doc/conf.py deleted file mode 100644 index d783c05d..00000000 --- a/doc/conf.py +++ /dev/null @@ -1,266 +0,0 @@ -# -*- coding: utf-8 -*- -# -# pywb documentation build configuration file, created by -# sphinx-quickstart on Thu Feb 27 21:08:12 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# adding project root to sys.path ('.' is not included by default) -sys.path.insert(0, os.path.dirname(__file__)) - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'pywb' -copyright = u'2014, Ilya Kreymer' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.2' -# The full version, including alpha/beta/rc tags. -release = '0.2' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pywbdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'pywb.tex', u'pywb Documentation', - u'Ilya Kreymer', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pywb', u'pywb Documentation', - [u'Ilya Kreymer'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'pywb', u'pywb Documentation', - u'Ilya Kreymer', 'pywb', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False diff --git a/doc/index.rst b/doc/index.rst deleted file mode 100644 index 932c0a60..00000000 --- a/doc/index.rst +++ /dev/null @@ -1,44 +0,0 @@ -.. pywb documentation master file, created by - sphinx-quickstart on Thu Feb 27 21:08:12 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to pywb's documentation! -================================ - -Pywb is a pure-Python implementation of Wayback Machine. - -.. toctree:: - :maxdepth: 2 - -Installing Pywb ---------------- - -Pywb QuickStart ---------------- - -The pywb tool suite currently includes two runnable applications in the -:mod:`pywb.apps` package:: - - python -m pywb.apps.wayback - -starts the full wayback on port 8080, and:: - - python -m pywb.apps.cdx_server - -starts standalone cdx server on port 8090 - -Customizing Pywb ----------------- - -Access Control -~~~~~~~~~~~~~~ - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/doc/modules.rst b/doc/modules.rst deleted file mode 100644 index 417a9afd..00000000 --- a/doc/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -pywb -==== - -.. toctree:: - :maxdepth: 4 - - pywb diff --git a/doc/pywb.apps.rst b/doc/pywb.apps.rst deleted file mode 100644 index 35c5f111..00000000 --- a/doc/pywb.apps.rst +++ /dev/null @@ -1,30 +0,0 @@ -pywb.apps package -================= - -Submodules ----------- - -pywb.apps.cdx_server module ---------------------------- - -.. automodule:: pywb.apps.cdx_server - :members: - :undoc-members: - :show-inheritance: - -pywb.apps.wayback module ------------------------- - -.. automodule:: pywb.apps.wayback - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pywb.apps - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.cdx.rst b/doc/pywb.cdx.rst deleted file mode 100644 index d972c4de..00000000 --- a/doc/pywb.cdx.rst +++ /dev/null @@ -1,76 +0,0 @@ -pywb.cdx package -================ - -Submodules ----------- - -pywb.cdx.cdxdomainspecific module ---------------------------------- - -.. automodule:: pywb.cdx.cdxdomainspecific - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.cdxobject module -------------------------- - -.. automodule:: pywb.cdx.cdxobject - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.cdxops module ----------------------- - -.. automodule:: pywb.cdx.cdxops - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.cdxserver module -------------------------- - -.. automodule:: pywb.cdx.cdxserver - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.cdxsource module -------------------------- - -.. automodule:: pywb.cdx.cdxsource - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.query module ---------------------- - -.. automodule:: pywb.cdx.query - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.cdx.zipnum module ----------------------- - -.. automodule:: pywb.cdx.zipnum - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pywb.cdx - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.core.rst b/doc/pywb.core.rst deleted file mode 100644 index 55d59e85..00000000 --- a/doc/pywb.core.rst +++ /dev/null @@ -1,65 +0,0 @@ -pywb.core package -================= - -Submodules ----------- - -pywb.core.cdx_handler module ----------------------------- - -.. automodule:: pywb.core.cdx_handler - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.core.handlers module -------------------------- - -.. automodule:: pywb.core.handlers - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.core.indexreader module ----------------------------- - -.. automodule:: pywb.core.indexreader - :members: - :undoc-members: - :show-inheritance: - -pywb.core.pywb_init module --------------------------- - -.. automodule:: pywb.core.pywb_init - :members: - :undoc-members: - :show-inheritance: - -pywb.core.replay_views module ------------------------------ - -.. automodule:: pywb.core.replay_views - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.core.views module ----------------------- - -.. automodule:: pywb.core.views - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pywb.core - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.framework.rst b/doc/pywb.framework.rst deleted file mode 100644 index 0a55d5f5..00000000 --- a/doc/pywb.framework.rst +++ /dev/null @@ -1,67 +0,0 @@ -pywb.framework package -====================== - -Submodules ----------- - -pywb.framework.archivalrouter module ------------------------------------- - -.. automodule:: pywb.framework.archivalrouter - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.framework.basehandlers module ----------------------------------- - -.. automodule:: pywb.framework.basehandlers - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.framework.proxy module ---------------------------- - -.. automodule:: pywb.framework.proxy - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.framework.wbexceptions module ----------------------------------- - -.. automodule:: pywb.framework.wbexceptions - :members: - :undoc-members: - :show-inheritance: - -pywb.framework.wbrequestresponse module ---------------------------------------- - -.. automodule:: pywb.framework.wbrequestresponse - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.framework.wsgi_wrappers module ------------------------------------ - -.. automodule:: pywb.framework.wsgi_wrappers - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - - -Module contents ---------------- - -.. automodule:: pywb.framework - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.perms.rst b/doc/pywb.perms.rst deleted file mode 100644 index 0de7fa72..00000000 --- a/doc/pywb.perms.rst +++ /dev/null @@ -1,38 +0,0 @@ -pywb.perms package -================== - -Subpackages ------------ - -.. toctree:: - - pywb.perms.test - -Submodules ----------- - -pywb.perms.perms_filter module ------------------------------- - -.. automodule:: pywb.perms.perms_filter - :members: - :undoc-members: - :show-inheritance: - -pywb.perms.perms_handler module -------------------------------- - -.. automodule:: pywb.perms.perms_handler - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - - -Module contents ---------------- - -.. automodule:: pywb.perms - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.rewrite.rst b/doc/pywb.rewrite.rst deleted file mode 100644 index 1feb66a0..00000000 --- a/doc/pywb.rewrite.rst +++ /dev/null @@ -1,79 +0,0 @@ -pywb.rewrite package -==================== - -Submodules ----------- - -pywb.rewrite.header_rewriter module ------------------------------------ - -.. automodule:: pywb.rewrite.header_rewriter - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.html_rewriter module ---------------------------------- - -.. automodule:: pywb.rewrite.html_rewriter - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.regex_rewriters module ------------------------------------ - -.. automodule:: pywb.rewrite.regex_rewriters - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.rewrite.rewrite_content module ------------------------------------ - -.. automodule:: pywb.rewrite.rewrite_content - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.rewrite_live module --------------------------------- - -.. automodule:: pywb.rewrite.rewrite_live - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.rewriterules module --------------------------------- - -.. automodule:: pywb.rewrite.rewriterules - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.url_rewriter module --------------------------------- - -.. automodule:: pywb.rewrite.url_rewriter - :members: - :undoc-members: - :show-inheritance: - -pywb.rewrite.wburl module -------------------------- - -.. automodule:: pywb.rewrite.wburl - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pywb.rewrite - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/pywb.warc.rst b/doc/pywb.warc.rst deleted file mode 100644 index c5369cbd..00000000 --- a/doc/pywb.warc.rst +++ /dev/null @@ -1,39 +0,0 @@ -pywb.warc package -================= - -Submodules ----------- - -pywb.warc.pathresolvers module ------------------------------- - -.. automodule:: pywb.warc.pathresolvers - :members: - :undoc-members: - :show-inheritance: - :special-members: __call__ - -pywb.warc.recordloader module ------------------------------ - -.. automodule:: pywb.warc.recordloader - :members: - :undoc-members: - :show-inheritance: - -pywb.warc.resolvingloader module --------------------------------- - -.. automodule:: pywb.warc.resolvingloader - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pywb.warc - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..a1c8d885 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,23 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = pywb +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +apidoc: + @sphinx-apidoc -f -T -o code ../pywb/ "../*test*" "../*git_hash*" + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/code/pywb.apps.rst b/docs/code/pywb.apps.rst new file mode 100644 index 00000000..e6008df5 --- /dev/null +++ b/docs/code/pywb.apps.rst @@ -0,0 +1,78 @@ +pywb\.apps package +================== + +Submodules +---------- + +pywb\.apps\.cli module +---------------------- + +.. automodule:: pywb.apps.cli + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.frontendapp module +------------------------------ + +.. automodule:: pywb.apps.frontendapp + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.live module +----------------------- + +.. automodule:: pywb.apps.live + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.rewriterapp module +------------------------------ + +.. automodule:: pywb.apps.rewriterapp + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.static\_handler module +---------------------------------- + +.. automodule:: pywb.apps.static_handler + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.warcserverapp module +-------------------------------- + +.. automodule:: pywb.apps.warcserverapp + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.wayback module +-------------------------- + +.. automodule:: pywb.apps.wayback + :members: + :undoc-members: + :show-inheritance: + +pywb\.apps\.wbrequestresponse module +------------------------------------ + +.. automodule:: pywb.apps.wbrequestresponse + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.apps + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.indexer.rst b/docs/code/pywb.indexer.rst new file mode 100644 index 00000000..8ac2104d --- /dev/null +++ b/docs/code/pywb.indexer.rst @@ -0,0 +1,30 @@ +pywb\.indexer package +===================== + +Submodules +---------- + +pywb\.indexer\.archiveindexer module +------------------------------------ + +.. automodule:: pywb.indexer.archiveindexer + :members: + :undoc-members: + :show-inheritance: + +pywb\.indexer\.cdxindexer module +-------------------------------- + +.. automodule:: pywb.indexer.cdxindexer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.indexer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.manager.rst b/docs/code/pywb.manager.rst new file mode 100644 index 00000000..d2ef00c1 --- /dev/null +++ b/docs/code/pywb.manager.rst @@ -0,0 +1,38 @@ +pywb\.manager package +===================== + +Submodules +---------- + +pywb\.manager\.autoindex module +------------------------------- + +.. automodule:: pywb.manager.autoindex + :members: + :undoc-members: + :show-inheritance: + +pywb\.manager\.manager module +----------------------------- + +.. automodule:: pywb.manager.manager + :members: + :undoc-members: + :show-inheritance: + +pywb\.manager\.migrate module +----------------------------- + +.. automodule:: pywb.manager.migrate + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.manager + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.recorder.rst b/docs/code/pywb.recorder.rst new file mode 100644 index 00000000..a7bdfd5c --- /dev/null +++ b/docs/code/pywb.recorder.rst @@ -0,0 +1,46 @@ +pywb\.recorder package +====================== + +Submodules +---------- + +pywb\.recorder\.filters module +------------------------------ + +.. automodule:: pywb.recorder.filters + :members: + :undoc-members: + :show-inheritance: + +pywb\.recorder\.multifilewarcwriter module +------------------------------------------ + +.. automodule:: pywb.recorder.multifilewarcwriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.recorder\.recorderapp module +---------------------------------- + +.. automodule:: pywb.recorder.recorderapp + :members: + :undoc-members: + :show-inheritance: + +pywb\.recorder\.redisindexer module +----------------------------------- + +.. automodule:: pywb.recorder.redisindexer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.recorder + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.rewrite.rst b/docs/code/pywb.rewrite.rst new file mode 100644 index 00000000..6e551540 --- /dev/null +++ b/docs/code/pywb.rewrite.rst @@ -0,0 +1,142 @@ +pywb\.rewrite package +===================== + +Submodules +---------- + +pywb\.rewrite\.content\_rewriter module +--------------------------------------- + +.. automodule:: pywb.rewrite.content_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.cookie\_rewriter module +-------------------------------------- + +.. automodule:: pywb.rewrite.cookie_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.cookies module +----------------------------- + +.. automodule:: pywb.rewrite.cookies + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.default\_rewriter module +--------------------------------------- + +.. automodule:: pywb.rewrite.default_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.header\_rewriter module +-------------------------------------- + +.. automodule:: pywb.rewrite.header_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.html\_insert\_rewriter module +-------------------------------------------- + +.. automodule:: pywb.rewrite.html_insert_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.html\_rewriter module +------------------------------------ + +.. automodule:: pywb.rewrite.html_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.jsonp\_rewriter module +------------------------------------- + +.. automodule:: pywb.rewrite.jsonp_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.regex\_rewriters module +-------------------------------------- + +.. automodule:: pywb.rewrite.regex_rewriters + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.rewrite\_amf module +---------------------------------- + +.. automodule:: pywb.rewrite.rewrite_amf + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.rewrite\_dash module +----------------------------------- + +.. automodule:: pywb.rewrite.rewrite_dash + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.rewrite\_hls module +---------------------------------- + +.. automodule:: pywb.rewrite.rewrite_hls + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.rewriteinputreq module +------------------------------------- + +.. automodule:: pywb.rewrite.rewriteinputreq + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.templateview module +---------------------------------- + +.. automodule:: pywb.rewrite.templateview + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.url\_rewriter module +----------------------------------- + +.. automodule:: pywb.rewrite.url_rewriter + :members: + :undoc-members: + :show-inheritance: + +pywb\.rewrite\.wburl module +--------------------------- + +.. automodule:: pywb.rewrite.wburl + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.rewrite + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/pywb.rst b/docs/code/pywb.rst similarity index 74% rename from doc/pywb.rst rename to docs/code/pywb.rst index dda76543..7344991b 100644 --- a/doc/pywb.rst +++ b/docs/code/pywb.rst @@ -7,13 +7,12 @@ Subpackages .. toctree:: pywb.apps - pywb.cdx - pywb.core - pywb.framework - pywb.perms + pywb.indexer + pywb.manager + pywb.recorder pywb.rewrite pywb.utils - pywb.warc + pywb.warcserver Module contents --------------- diff --git a/doc/pywb.utils.rst b/docs/code/pywb.utils.rst similarity index 57% rename from doc/pywb.utils.rst rename to docs/code/pywb.utils.rst index 8fdd5e49..aebce3a3 100644 --- a/doc/pywb.utils.rst +++ b/docs/code/pywb.utils.rst @@ -1,68 +1,67 @@ -pywb.utils package -================== +pywb\.utils package +=================== Submodules ---------- -pywb.utils.binsearch module ---------------------------- +pywb\.utils\.binsearch module +----------------------------- .. automodule:: pywb.utils.binsearch :members: :undoc-members: :show-inheritance: -pywb.utils.bufferedreaders module ---------------------------------- - -.. automodule:: pywb.utils.bufferedreaders - :members: - :undoc-members: - :show-inheritance: - -pywb.utils.canonicalize module ------------------------------- +pywb\.utils\.canonicalize module +-------------------------------- .. automodule:: pywb.utils.canonicalize :members: :undoc-members: :show-inheritance: - :special-members: __call__ -pywb.utils.dsrules module -------------------------- +pywb\.utils\.format module +-------------------------- -.. automodule:: pywb.utils.dsrules +.. automodule:: pywb.utils.format :members: :undoc-members: :show-inheritance: -pywb.utils.loaders module -------------------------- +pywb\.utils\.geventserver module +-------------------------------- + +.. automodule:: pywb.utils.geventserver + :members: + :undoc-members: + :show-inheritance: + +pywb\.utils\.io module +---------------------- + +.. automodule:: pywb.utils.io + :members: + :undoc-members: + :show-inheritance: + +pywb\.utils\.loaders module +--------------------------- .. automodule:: pywb.utils.loaders :members: :undoc-members: :show-inheritance: -pywb.utils.statusandheaders module ----------------------------------- - -.. automodule:: pywb.utils.statusandheaders - :members: - :undoc-members: - :show-inheritance: - -pywb.utils.timeutils module +pywb\.utils\.memento module --------------------------- -.. automodule:: pywb.utils.timeutils +.. automodule:: pywb.utils.memento :members: :undoc-members: :show-inheritance: -pywb.utils.wbexception module ------------------------------ +pywb\.utils\.wbexception module +------------------------------- .. automodule:: pywb.utils.wbexception :members: diff --git a/docs/code/pywb.warcserver.index.rst b/docs/code/pywb.warcserver.index.rst new file mode 100644 index 00000000..d1b5fdec --- /dev/null +++ b/docs/code/pywb.warcserver.index.rst @@ -0,0 +1,70 @@ +pywb\.warcserver\.index package +=============================== + +Submodules +---------- + +pywb\.warcserver\.index\.aggregator module +------------------------------------------ + +.. automodule:: pywb.warcserver.index.aggregator + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.cdxobject module +----------------------------------------- + +.. automodule:: pywb.warcserver.index.cdxobject + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.cdxops module +-------------------------------------- + +.. automodule:: pywb.warcserver.index.cdxops + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.fuzzymatcher module +-------------------------------------------- + +.. automodule:: pywb.warcserver.index.fuzzymatcher + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.indexsource module +------------------------------------------- + +.. automodule:: pywb.warcserver.index.indexsource + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.query module +------------------------------------- + +.. automodule:: pywb.warcserver.index.query + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.index\.zipnum module +-------------------------------------- + +.. automodule:: pywb.warcserver.index.zipnum + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.warcserver.index + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.warcserver.resource.rst b/docs/code/pywb.warcserver.resource.rst new file mode 100644 index 00000000..e29d93de --- /dev/null +++ b/docs/code/pywb.warcserver.resource.rst @@ -0,0 +1,46 @@ +pywb\.warcserver\.resource package +================================== + +Submodules +---------- + +pywb\.warcserver\.resource\.blockrecordloader module +---------------------------------------------------- + +.. automodule:: pywb.warcserver.resource.blockrecordloader + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.resource\.pathresolvers module +------------------------------------------------ + +.. automodule:: pywb.warcserver.resource.pathresolvers + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.resource\.resolvingloader module +-------------------------------------------------- + +.. automodule:: pywb.warcserver.resource.resolvingloader + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.resource\.responseloader module +------------------------------------------------- + +.. automodule:: pywb.warcserver.resource.responseloader + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.warcserver.resource + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/code/pywb.warcserver.rst b/docs/code/pywb.warcserver.rst new file mode 100644 index 00000000..0498fabe --- /dev/null +++ b/docs/code/pywb.warcserver.rst @@ -0,0 +1,70 @@ +pywb\.warcserver package +======================== + +Subpackages +----------- + +.. toctree:: + + pywb.warcserver.index + pywb.warcserver.resource + +Submodules +---------- + +pywb\.warcserver\.basewarcserver module +--------------------------------------- + +.. automodule:: pywb.warcserver.basewarcserver + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.handlers module +--------------------------------- + +.. automodule:: pywb.warcserver.handlers + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.http module +----------------------------- + +.. automodule:: pywb.warcserver.http + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.inputrequest module +------------------------------------- + +.. automodule:: pywb.warcserver.inputrequest + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.upstreamindexsource module +-------------------------------------------- + +.. automodule:: pywb.warcserver.upstreamindexsource + :members: + :undoc-members: + :show-inheritance: + +pywb\.warcserver\.warcserver module +----------------------------------- + +.. automodule:: pywb.warcserver.warcserver + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pywb.warcserver + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..835f8b5d --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# pywb documentation build configuration file, created by +# sphinx-quickstart on Thu Sep 21 01:58:55 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'pywb' +copyright = 'A Webrecorder Project, Ilya Kreymer, Rhizome' +author = 'Ilya Kreymer' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '2.0' +# The full version, including alpha/beta/rc tags. +release = '2.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +#html_sidebars = { +# '**': [ +# 'about.html', +# 'navigation.html', +# 'relations.html', # needs 'show_related': True theme option to display +# 'searchbox.html', +# 'donate.html', +# ] +#} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pywbdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'pywb.tex', 'pywb Documentation', + 'Ilya Kreymer', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pywb', 'pywb Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'pywb', 'pywb Documentation', + author, 'pywb', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..b318aae1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,27 @@ +.. pywb documentation master file, created by + sphinx-quickstart on Thu Sep 21 01:58:55 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Webrecorder pywb documentation! +================================ + +Webrecorder (:mod:`pywb`) toolkit is a full-featured, advanced web archiving capture and replay framework for python. +It provides command-line tools and an extensible framework for high-fidelity web archive access and creation. + +.. toctree:: + :maxdepth: 2 + + manual/intro + manual/usage + manual/configuring + manual/index + code/pywb + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..d5a9246e --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=pywb + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/manual/configuring.rst b/docs/manual/configuring.rst new file mode 100644 index 00000000..a303baf4 --- /dev/null +++ b/docs/manual/configuring.rst @@ -0,0 +1,269 @@ +Configuring the Web Archive +=========================== + +pywb offers an extensible YAML based configuration format via a main ``config.yaml`` at the root of each web archive. + +Framed vs Frameless Replay vs HTTPS proxy +----------------------------------------- + +pywb supports several modes for serving archived web content. + +With **framed replay**, the archived content is loaded into an iframe, and a top frame UI provides info and metadata. +In this mode, the top frame url is for example, ``http://my-archive.example.com//http://example.com/`` while +the actual content is served at ``http://my-archive.example.com//mp_/http://example.com/`` + + +With **frameless replay**, the archived content is loaded directly, and a banner UI is injected into the page. + +In this mode, the content is served directly at ``http://my-archive.example.com//http://example.com/`` + +(pywb can also supports HTTP/S **proxy mode** which requires additional setup. See :ref:`https-proxy` for more details). + +For security reasons, we recommend running pywb in framed mode, because a malicious site +`could tamper with the banner `_ + +However, for certain situations, frameless replay made be appropriate. + +To disable framed replay add: + +``framed_replay: false`` to your config.yaml + + +Directory Structure +------------------- + +The pywb system assumes the following default directory structure for a web archive:: + + +-- config.yaml (optional) + | + +-- templates (optional) + | + +-- static (optional) + | + +-- collections + | + +-- + | + +-- archives + | | + | +-- (WARC or ARC files here) + | + +-- indexes + | | + | +-- (CDXJ index files here) + | + +-- templates + | | + | +-- (optional html templates here) + | + +-- static + | + +-- (optional custom static assets here) + + +If running with default settings, the ``config.yaml`` can be omitted. + +It is possible to config these paths in the config.yaml +The following are some of the implicit default settings which can be customized:: + + collections_root: collections + archive_paths: archive + index_paths: indexes + +(For a complete list of defaults, see the ``pywb/default_config.yaml`` file for reference) + +Index Paths +^^^^^^^^^^^ + +The ``index_paths`` key defines the subdirectory for index files (usually CDXJ) and determine the contents of each archive collection. + +The index files usually contain a pointer to a WARC file, but not the absolute path. + +Archive Paths +^^^^^^^^^^^^^ + +The ``archive_paths`` key indicates how pywb will resolve WARC files listed in the index. + +For example, it is possible to configure multiple archive paths:: + + archive_paths: + - archive + - http://remote-bakup.example.com/collections/ + +When resolving a ``example.warc.gz``, pywb will then check (in order): + + * First, ``collections//example.warc.gz`` + * Then, ``http://remote-backup.example.com/collections//example.warc.gz`` (if first lookup unsuccessful) + + +Custom Defined Collections +-------------------------- + +While pywb can detect automatically collections following the above directory structure, +it may be useful to declare custom collections explicitly. + +In addition, several "special" collection definitions are possible. + +All custom defined collections are placed under the ``collections`` key in ``config.yaml`` + + +Live Web Collection +^^^^^^^^^^^^^^^^^^^ + +The live web collection proxies all data to the live web, and can be defined as follows:: + + collections: + live: $live + +This configures the ``/live/`` route to point to the live web. + +(As a shortcut, ``wayback --live`` adds this collection via cli w/o modifiying the config.yaml) + +This collection can be useful for testing, or even more powerful, when combined with recording. + + +Auto "All" Aggregate Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The aggregate all collections automatically aggregates data from all collections in the ``collections`` directory:: + + collections: + all: $all + +Accessing ``/all/`` will cause an aggregate lookup within the collections directory. + +Note: It is not (yet) possible to exclude collections from the all collection, although "special" collections are not included. + + +Generic Collection Definitions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The collection definition syntax allows for explicitly setting the index, archive paths +and all other templates, per collection, for example:: + + collections: + custom: + index: ./path/to/indexes + resource: ./some/other/path/to/archive/ + query_html: ./path/to/templates/query.html + +This configuration supports the full Warcserver config syntax, including +remote archives, aggregation and fallback sequences (link) + +This format also makes it easier to move legacy collections that have unique path requirements. + +Root Collection +^^^^^^^^^^^^^^^ + +It is also possible to define a "root" collection, for example, accessible at ``http://my-archive.example.com/`` +Such a collection must be defined explicitly using the ``$root`` as collection name:: + + collections: + $root: + index: ./path/to/indexes + resource: ./path/to/archive/ + +Note: When a root collection is set, no other collections are currently accessible, they are ignored. + + +Recording Mode +-------------- + +TODO + +.. _https-proxy: + +HTTP/S Proxy Mode +----------------- + +TODO + +UI Customizations +----------------- + +pywb supports UI customizations, either for an entire archive, +or per-collection. + +Static Files +^^^^^^^^^^^^ + +The replay server will automatically support static files placed under the following directories: + +* Files under the root ``static`` directory can be accessed via ``http://my-archive.example.com/static/`` + +* Files under the per-collection ``./collections//static`` directory can be accessed via ``http://my-archive.example.com/static/_//`` + +Templates +^^^^^^^^^ + +pywb users Jinja2 templates to render HTML to render the HTML for all aspects of the application. +A version placed in the ``templates`` directory, either in the root or per collection, will override that template. + +To copy the default pywb template to the template directory run: + +``wb-manager template --add search_html`` + +The following templates are available: + + * ``home.html`` -- Home Page Template, used for ``http://my-archive.example.com/`` + + * ``search.html`` -- Collection Template, used for each collection page ``http://my-archive.example.com//`` + + * ``query.html`` -- Capture Query Page for a given url, used for ``http://my-archive.example.com/`` + +Error Pages: + + * ``not_found.html`` -- Page to show when a url is not found in the archive + + * ``error.html`` -- Generic Error Page for any error (except not found) + +Replay and Banner templates: + + * ``frame_insert.html`` -- Top-frame for framed replay mode (not used with frameless mode) + + * ``head_insert.html`` -- Rewriting code injected into ```` of each replayed page. + This template includes the banner template and itself should generally not need to be modified. + + * ``banner.html`` -- The banner used for frameless replay. Can be set to blank to disable the banner. + + +Custom Outer Replay Frame +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The top-frame used for framed replay can be replaced or augmented +by modifiying the ``frame_insert.html``. + +To start with modifiying the default outer page, you can add it to the current +templates directory by running ``wb-frame template --add frame_insert.html`` + +To initialize the replay, the outer page should include ``wb_frame.js``, +create an ``