update run-uwsgi.sh and add run-gunicorn.sh

update README and INSTALL, fix typo
only list wb handlers on home page by default
pep8 fixes

INSTALL.rst

@@ -55,7 +55,7 @@ To start a pywb with sample data:
 
 3. Run ``wayback`` (shorthand for ``python -m pywb.apps.wayback``) to start the pywb wayback server with reference WSGI implementation.
 
-OR run ``run-uwsgi.sh`` to start with uWSGI (see below for more info).
+OR run ``run-uwsgi.sh`` or ``run-gunicorn.sh`` to start with uWSGI or gunicorn (see below for more info).
 
 4. Test pywb in your browser! (pywb is set to run on port 8080 by
    default).
@@ -71,18 +71,17 @@ If everything worked, the following pages should be loading (served from
 | ``http://iana.org``    | http://localhost:8080/pywb/iana.org    | http://localhost:8080/pywb/\*/iana.org     |
 +------------------------+----------------------------------------+--------------------------------------------+
 
-uWSGI startup script
+uWSGI and gunicorn startup scripts
-^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A sample uWSGI start up script, ``run-uwsgi.sh`` which assumes a default
+pywb includes sample uWSGI and gunicorn scripts ``run-uwsgi.sh`` and
-uWSGI installation is provided as well.
+``run-gunicorn.sh`` which pip install uwsgi and gunicorn and attempt to launch
-
+the wsgi app with those containers.
-Currently, uWSGI is not installed automatically with this distribution,
-but it is recommended for production environments.
 
 Please see `uWSGI
 Installation <http://uwsgi-docs.readthedocs.org/en/latest/Install.html>`_
-for more details on installing uWSGI.
+and `Gunicorn QuickStart <http://gunicorn.org/>`_
+for more details on installing these containers.
 
 Vagrant
 ~~~~~~~

README.rst

@@ -50,7 +50,7 @@ Given an archive of warcs at ``myarchive/warcs``
 2. Run ``cdx-indexer --sort myarchive/cdx myarchive/warcs`` to generate .cdx files for each
    warc/arc file in ``myarchive/warcs``
 
-3. Edit ``<https://github.com/ikreymer/pywb/blob/develop/config.yaml>`` to contain the following. You may replace ``pywb`` with
+3. Edit **config.yaml** to contain the following. You may replace ``pywb`` with
    a name of your choice -- it will be the path to your collection. (Multiple collections can be added
    for different sets of .cdx files as well)
 
@@ -67,7 +67,7 @@ Given an archive of warcs at ``myarchive/warcs``
    If your archives contain ``http://my-archive-page.example.com``, all captures should be accessible
    by browsing to http://localhost:8080/pywb/\*/my-archived-page.example.com
 
-   (You can also ./run-uwsgi.sh for running with those WSGI containers)
+   (You can also use ``run-uwsgi.sh`` or ``run-gunicorn.sh`` to launch using those WSGI containers)
 
 
 See `INSTALL.rst <https://github.com/ikreymer/pywb/blob/develop/INSTALL.rst>`_ for additional installation info.

pywb/cdx/cdxobject.py

@@ -1,4 +1,4 @@
-try: # pragma: no cover
+try:  # pragma: no cover
     from collections import OrderedDict
 except ImportError:  # pragma: no cover
     from ordereddict import OrderedDict

pywb/core/handlers.py

@@ -47,28 +47,24 @@ class WBHandler(WbUrlHandler):
             return WbResponse.text_response('No Lookup Url Specified')
 
     def __str__(self):
-        return 'WBHandler: ' + str(self.index_reader) + ', ' + str(self.replay)
+        return 'Web Archive Replay Handler'
 
 
 #=================================================================
 # Static Content Handler
 #=================================================================
 class StaticHandler(BaseHandler):
-    def __init__(self, static_path, pkg='pywb'):
+    def __init__(self, static_path):
         mimetypes.init()
 
         self.static_path = static_path
-        self.pkg = pkg
+        self.block_loader = BlockLoader()
 
     def __call__(self, wbrequest):
         full_path = self.static_path + wbrequest.wb_url_str
 
         try:
-            #if full_path.startswith('.') or full_path.startswith('file://'):
+            data = self.block_loader.load(full_path)
-            #    data = open(full_path, 'rb')
-            #else:
-            #    data = pkgutil.get_data(self.pkg, full_path)
-            data = BlockLoader().load(full_path)
 
             if 'wsgi.file_wrapper' in wbrequest.env:
                 reader = wbrequest.env['wsgi.file_wrapper'](data)

pywb/core/views.py

@@ -2,6 +2,8 @@ from pywb.utils.timeutils import timestamp_to_datetime
 from pywb.framework.wbrequestresponse import WbResponse
 from pywb.framework.memento import make_timemap, LINK_FORMAT
 
+from pywb.core.handlers import WBHandler
+
 import urlparse
 import logging
 
@@ -54,6 +56,14 @@ def request_hostname(env):
     return env.get('HTTP_HOST', 'localhost')
 
 
+@template_filter()
+def is_wb_handler(obj):
+    if not hasattr(obj, 'handler'):
+        return False
+
+    return isinstance(obj.handler, WBHandler)
+
+
 #=================================================================
 class J2TemplateView:
     def __init__(self, filename):

pywb/ui/index.html

@@ -3,7 +3,9 @@
 The following archive collections are available:
 
 <ul>
-{% for route in routes  %}
+{% for route in routes %}
+  {% if route | is_wb_handler %}
   <li><a href="{{ '/' + route.path }}">{{ '/' + route.path }}</a>: {{ route | string }}</li>
+  {% endif %} 
 {% endfor %}
 </ul>

pywb/warc/archiveindexer.py

@@ -344,9 +344,11 @@ def remove_ext(filename):
 
     return filename
 
+
 def cdx_filename(filename):
     return remove_ext(filename) + '.cdx'
 
+
 def index_to_dir(inputs, output, sort):
     for fullpath, filename in iter_file_or_dir(inputs):
 

run-gunicorn.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+pip install gunicorn
+
+if [ $? -ne 0 ]; then
+    "uwsgi install failed"
+    exit 1
+fi
+
+export PYWB_CONFIG_FILE=config.yaml
+gunicorn -w 4 pywb.apps.wayback -b 0.0.0.0:8080

run-uwsgi.sh

@@ -1,39 +1,20 @@
 #!/bin/sh
 
+# requires uwsgi
+pip install uwsgi
+
+if [ $? -ne 0 ]; then
+    "uwsgi install failed"
+    exit 1
+fi
+
+
 mypath=$(cd `dirname $0` && pwd)
 
-# Set a different config file
+params="$mypath/uwsgi.ini"
-#export 'PYWB_CONFIG_FILE=myconfig.yaml'
 
-app="pywb.apps.wayback"
-
-params="--http-socket :8080 -b 65536"
-#params="--static-map /static=$mypath/static --http-socket :8080 -b 65536"
-
-if [ -z "$1" ]; then
-    # Standard root config
-    params="$params --wsgi $app"
-else
-    # run with --mount to specify a non-root context
-    # requires a file not a package, so creating a mount_run.py to load the package
-    echo "#!/bin/python\n" > $mypath/mount_run.py
-    echo "import $app\napplication = $app.application" >> $mypath/mount_run.py
-    params="$params --mount $1=mount_run.py --no-default-app --manage-script-name"
-fi
-
-# Support for virtualenv
 if [ -n "$VIRTUAL_ENV" ] ; then
     params="$params -H $VIRTUAL_ENV"
 fi
 
-# Support for default, non-virtualenv path on OS X
+uwsgi $params
-osx_uwsgi_path="/System/Library/Frameworks/Python.framework/Versions/2.7/bin/uwsgi"
-
-if [ -e "$osx_uwsgi_path" ]; then
-    uwsgi=$osx_uwsgi_path
-else
-    uwsgi="uwsgi"
-fi
-
-$uwsgi $params
-

uwsgi.ini

@@ -0,0 +1,13 @@
+[uwsgi]
+if-not-env = PORT
+http-socket = :8080
+endif =
+
+master = true
+processes = 10
+buffer-size = 65536
+die-on-term = true
+
+# specify config file here
+env = PYWB_CONFIG_FILE=config.yaml
+wsgi = pywb.apps.wayback