first pass on html rendering via jinja, support for query (cdx) rendering

pywb/indexreader.py

@@ -2,6 +2,7 @@ import urllib
 import urllib2
 import wbexceptions
 import itertools
+from collections import OrderedDict
 
 from wbarchivalurl import ArchivalUrl
 
@@ -78,7 +79,7 @@ class RemoteCDXServer:
         }[wburl.type]
 
 
-class CDXCaptureResult(dict):
+class CDXCaptureResult(OrderedDict):
     CDX_FORMATS = [
         # Public CDX Format
         ["urlkey","timestamp","original","mimetype","statuscode","digest","length"],
@@ -99,6 +100,8 @@ class CDXCaptureResult(dict):
         ]
 
     def __init__(self, cdxline):
+        OrderedDict.__init__(self)
+
         cdxline = cdxline.rstrip()
         fields = cdxline.split(' ')
 

pywb/query.py

@@ -3,6 +3,8 @@ import utils
 import wbrequestresponse
 import wbexceptions
 
+from jinja2 import Environment, FileSystemLoader
+
 class QueryHandler:
     def __init__(self, cdxserver = None):
         if not cdxserver:
@@ -23,13 +25,50 @@ class QueryHandler:
 
         cdxlines = utils.peek_iter(cdxlines)
 
-        if cdxlines is not None:
+        if cdxlines is None:
-            return wbrequestresponse.WbResponse.text_stream(cdxlines)
+            raise wbexceptions.NotFoundException('WB Does Not Have Url: ' + wburl.url)
+
+        cdxlines = self.filterCdx(wbrequest, cdxlines)
+
+        # Output raw cdx stream
+        return wbrequestresponse.WbResponse.text_stream(cdxlines)
+
+    def filterCdx(self, wbrequest, cdxlines):
+        # Subclasses may wrap cdxlines iterator in a filter
+        return cdxlines
+
+
+class J2QueryRenderer:
+    def __init__(self, template_dir, template_file):
+        self.template_file = template_file
+
+        self.jinja_env = Environment(loader = FileSystemLoader(template_dir), trim_blocks = True)
+
+    def __call__(self, wbrequest, query_response):
+        cdxlines = query_response.body
+
+        def parse_cdx():
+            for cdx in cdxlines:
+                try:
+                    cdx = indexreader.CDXCaptureResult(cdx)
+                    yield cdx
+
+                except wbexceptions.InvalidCDXException:
+                    import traceback
+                    traceback.print_exc()
+                    pass
+
+
+        template = self.jinja_env.get_template(self.template_file)
+        response = template.render(cdxlines = parse_cdx(),
+                                   url = wbrequest.wb_url.url,
+                                   prefix = wbrequest.wb_prefix)
+
+        return wbrequestresponse.WbResponse.text_response(str(response), content_type = 'text/html')
 
-        raise wbexceptions.NotFoundException('WB Does Not Have Url: ' + wburl.url)
 
 ## ===========
-## Simple handlers for debuging
+## Simple handlers for debugging
 class EchoEnv:
     def __call__(self, wbrequest):
         return wbrequestresponse.WbResponse.text_response(str(wbrequest.env))

pywb/replay.py

@@ -17,16 +17,17 @@ import wbexceptions
 
 #=================================================================
 class WBHandler:
-    def __init__(self, query, replay):
+    def __init__(self, query, replay, htmlquery = None):
         self.query = query
         self.replay = replay
+        self.htmlquery = htmlquery
 
     def __call__(self, wbrequest):
         with utils.PerfTimer(wbrequest.env.get('X_PERF'), 'query') as t:
             query_response = self.query(wbrequest)
 
         if (wbrequest.wb_url.type == ArchivalUrl.QUERY) or (wbrequest.wb_url.type == ArchivalUrl.URL_QUERY):
-            return query_response
+            return self.htmlquery(wbrequest, query_response) if self.htmlquery else query_response
 
         with utils.PerfTimer(wbrequest.env.get('X_PERF'), 'replay') as t:
             return self.replay(wbrequest, query_response)

pywb/wbrequestresponse.py

@@ -119,12 +119,12 @@ class WbResponse:
         self.body = value
 
     @staticmethod
-    def text_stream(text, status = '200 OK'):
+    def text_stream(text, status = '200 OK', content_type = 'text/plain'):
-        return WbResponse(StatusAndHeaders(status, [('Content-Type', 'text/plain')]), value = text)
+        return WbResponse(StatusAndHeaders(status, [('Content-Type', content_type)]), value = text)
 
     @staticmethod
-    def text_response(text, status = '200 OK'):
+    def text_response(text, status = '200 OK', content_type = 'text/plain'):
-        return WbResponse(StatusAndHeaders(status, [('Content-Type', 'text/plain')]), value = [text])
+        return WbResponse(StatusAndHeaders(status, [('Content-Type', content_type)]), value = [text])
 
     @staticmethod
     def redir_response(location, status = '302 Redirect'):

setup.py

@@ -11,7 +11,7 @@ setuptools.setup(name='pywb',
         long_description=open('README.md').read(),
         license='GPL',
         packages=['pywb'],
-        install_requires=['uwsgi', 'rfc3987', 'chardet', 'redis'],
+        install_requires=['uwsgi', 'rfc3987', 'chardet', 'redis', 'jinja2'],
         # test_suite='?',   # not sure how to run doctests here
         zip_safe=False)
 

ui/query.html

@@ -0,0 +1,14 @@
+<body>
+  <b><span id="count"></span>&nbsp;Captures of {{ url }}</b>
+  <table id="captures">
+  {% for cdx in cdxlines %}
+  <tr>
+    <td><a href="{{ prefix}}{{ cdx.timestamp }}/{{ url }}">{{ cdx.timestamp }}</a></td>
+    <td><a href="https://archive.org/details/{{ cdx['filename'] }}">{{ cdx['filename'] }}</a></td>
+  </tr>
+  {% endfor %}
+  </table>
+  <script>
+document.getElementById("count").innerHTML = document.getElementById("captures").getElementsByTagName("tr").length
+ </script>
+</body>