fixes! Fix typos, in html parsing, fix base, support attrs w/o values

pywb/archiveloader.py

@@ -7,6 +7,7 @@ import urllib2
 import StringIO
 import urlparse
 import collections
+import wbexceptions
 
 #=================================================================
 class HttpStreamLoader:
@@ -143,9 +144,9 @@ class ArchiveLoader:
             headers = [('Content-Type', utils.get_header(parsed.headers, 'Content-Type'))]
 
         # special case: http 0.9 response, no status or headers
-        elif recType == 'response' and (';version=0.9' in utils.get_header(parsed.headers, 'Content-Type')):
+        #elif recType == 'response' and (';version=0.9' in utils.get_header(parsed.headers, 'Content-Type')):
-            statusline = '200 OK'
+        #    statusline = '200 OK'
-            headers = []
+        #    headers = []
 
         # response record: parse HTTP status and headers!
         else:
@@ -202,13 +203,13 @@ class LineReader:
 
             self.buff = StringIO.StringIO(data)
 
-    def read(self):
+    def read(self, length = None):
         self._fillbuff()
-        return self.buff.read()
+        return self.buff.read(length)
 
-    def readline(self):
+    def readline(self, length = None):
         self._fillbuff()
-        return self.buff.readline()
+        return self.buff.readline(length)
 
     def close(self):
         if self.stream:

pywb/replay.py

@@ -117,7 +117,7 @@ class ReplayHandler(object):
             if fullUrl:
                 return fullUrl
 
-        raise exceptions.UnresolvedArchiveFileException('Archive File Not Found: ' + cdx.filename)
+        raise wbexceptions.UnresolvedArchiveFileException('Archive File Not Found: ' + filename)
 
 
 #=================================================================
@@ -125,10 +125,10 @@ class RewritingReplayHandler(ReplayHandler):
 
 
     REWRITE_TYPES = {
-        'html': ('text/html', 'application/xhtml'),
+        'html': ['text/html', 'application/xhtml'],
-        'css':  ('text/css'),
+        'css':  ['text/css'],
-        'js':   ('text/javascript', 'application/javascript', 'application/x-javascript'),
+        'js':   ['text/javascript', 'application/javascript', 'application/x-javascript'],
-        'xml':  ('/xml', '+xml', '.xml', '.rss'),
+        'xml':  ['/xml', '+xml', '.xml', '.rss'],
     }
 
 
@@ -145,11 +145,10 @@ class RewritingReplayHandler(ReplayHandler):
         self.headInsert = headInsert
 
 
-    def _canonContentType(self, contentType):
+    def _textContentType(self, contentType):
-        for type, mimelist in self.REWRITE_TYPES.iteritems():
+        for ctype, mimelist in self.REWRITE_TYPES.iteritems():
-            for mime in mimelist:
+            if any ((mime in contentType) for mime in mimelist):
-                if mime in contentType:
+                return ctype
-                    return type
 
         return None
 
@@ -168,13 +167,13 @@ class RewritingReplayHandler(ReplayHandler):
             return response
 
         contentType = utils.get_header(response.headersList, 'Content-Type')
-
+        
-        canonType = self._canonContentType(contentType)
+        textType = self._textContentType(contentType) if contentType else None
-
+        
-        (newHeaders, remHeaders) = self._rewriteHeaders(response.headersList, (canonType is not None))
+        (newHeaders, remHeaders) = self._rewriteHeaders(response.headersList, urlrewriter, textType is not None)
 
         # binary type, just send through
-        if canonType is None:
+        if textType is None:
             response.headersList = newHeaders
             return response
 
@@ -186,19 +185,19 @@ class RewritingReplayHandler(ReplayHandler):
         if (utils.contains_header(remHeaders, ('Content-Encoding', 'gzip'))):
             stream = archiveloader.LineStream(stream, decomp = zlib.decompressobj(16 + zlib.MAX_WBITS))
 
-        return self._rewriteContent(canonType, urlrewriter, stream, newHeaders, response)
+        return self._rewriteContent(textType, urlrewriter, stream, newHeaders, response)
 
     # TODO: first non-streaming attempt, probably want to stream
-    def _rewriteContent(self, canonType, urlrewriter, stream, newHeaders, origResponse):
+    def _rewriteContent(self, textType, urlrewriter, stream, newHeaders, origResponse, encoding = 'utf-8'):
-        if canonType == 'html':
+        if textType == 'html':
             out = StringIO.StringIO()
             htmlrewriter = wbhtml.WBHtml(urlrewriter, out, self.headInsert)
 
             try:
-                buff = stream.read()
+                buff = stream.read()#.decode(encoding)
                 while buff:
                     htmlrewriter.feed(buff)
-                    buff = stream.read()
+                    buff = stream.read()#.decode(encoding)
 
                 htmlrewriter.close()
 
@@ -207,12 +206,13 @@ class RewritingReplayHandler(ReplayHandler):
 
             finally:
                 value = [out.getvalue()]
+                newHeaders.append(('Content-Length', str(len(value[0]))))
                 out.close()
 
         else:
-            if canonType == 'css':
+            if textType == 'css':
                 rewriter = regexmatch.CSSRewriter(urlrewriter)
-            elif canonType == 'js':
+            elif textType == 'js':
                 rewriter = regexmatch.JSRewriter(urlrewriter)
 
             def gen():
@@ -231,7 +231,7 @@ class RewritingReplayHandler(ReplayHandler):
 
 
 
-    def _rewriteHeaders(self, headers, stripEncoding = False):
+    def _rewriteHeaders(self, headers, urlrewriter, stripEncoding = False):
         newHeaders = []
         removedHeaders = []
 

pywb/wbexceptions.py

@@ -34,7 +34,7 @@ class UnknownLoaderProtocolException(CaptureException):
     pass
 
 class InvalidArchiveRecordException(CaptureException):
-    def __init__(msg, errList = None):
+    def __init__(self, msg, errList = None):
         super(InvalidArchiveRecordException, self).__init__(msg)
         self.errList = errList
 

pywb/wbhtml.py

@@ -19,6 +19,13 @@ class WBHtml(HTMLParser):
     >>> parse('<body x="y"><img src="/img.gif"/><br/></body>')
     <body x="y"><img src="/web/20131226101010im_/http://example.com/img.gif"/><br/></body>
 
+    >>> parse('<input "selected"><img src></div>')
+    <input "selected"><img src></div>
+
+    >>> parse('<html><head><base href="http://example.com/some/path/index.html"/>')
+    <html><head><base href="/web/20131226101010/http://example.com/some/path/index.html"/>
+   
+    # Meta tag
     >>> parse('<META http-equiv="refresh" content="10; URL=/abc/def.html">')
     <meta http-equiv="refresh" content="10; URL=/web/20131226101010/http://example.com/abc/def.html">
 
@@ -35,8 +42,8 @@ class WBHtml(HTMLParser):
     >>> parse('<script>/*<![CDATA[*/window.location = "http://example.com/a/b/c.html;/*]]>*/"</script>')
     <script>/*<![CDATA[*/window.WB_wombat_location = "/web/20131226101010/http://example.com/a/b/c.html;/*]]>*/"</script>
 
-    >>> parse('<div style="background: url(\'abc.html\')" onclick="location = \'redirect.html\'"></div>')
+    >>> parse('<div style="background: url(\'abc.html\')" onblah onclick="location = \'redirect.html\'"></div>')
-    <div style="background: url('/web/20131226101010/http://example.com/some/path/abc.html')" onclick="WB_wombat_location = 'redirect.html'"></div>
+    <div style="background: url('/web/20131226101010/http://example.com/some/path/abc.html')" onblah onclick="WB_wombat_location = 'redirect.html'"></div>
 
     >>> parse('<style>@import "styles.css" .a { font-face: url(\'myfont.ttf\') }</style>')
     <style>@import "/web/20131226101010/http://example.com/some/path/styles.css" .a { font-face: url('/web/20131226101010/http://example.com/some/path/myfont.ttf') }</style>
@@ -130,14 +137,14 @@ class WBHtml(HTMLParser):
     # ===========================
 
     def _rewriteURL(self, value, mod = None):
-        return self.rewriter.rewrite(value, mod)
+        return self.rewriter.rewrite(value, mod) if value else None
 
 
     def _rewriteCSS(self, cssContent):
-        return self.cssRewriter.replaceAll(cssContent)
+        return self.cssRewriter.replaceAll(cssContent) if cssContent else None
 
     def _rewriteScript(self, scriptContent):
-        return self.jsRewriter.replaceAll(scriptContent)
+        return self.jsRewriter.replaceAll(scriptContent) if scriptContent else None
 
     def hasAttr(self, tagAttrs, attr):
         name, value = attr
@@ -147,14 +154,9 @@ class WBHtml(HTMLParser):
         return False
 
     def rewriteTagAttrs(self, tag, tagAttrs, isStartEnd):
-        # special case: base tag
+        
-        if (tag == 'base'):
-            newBase = tagAttrs.get('href')
-            if newBase:
-                self.rewriter.setBaseUrl(newBase[1])
-
         # special case: script or style parse context
-        elif (tag in WBHtml.STATE_TAGS) and (self._wbParseContext == None):
+        if (tag in WBHtml.STATE_TAGS) and (self._wbParseContext == None):
             self._wbParseContext = tag
 
         # special case: head insertion, non-head tags
@@ -176,7 +178,7 @@ class WBHtml(HTMLParser):
             attrName, attrValue = attr
 
             # special case: inline JS/event handler
-            if attrValue.startswith('javascript:') or attrName.startswith("on"):
+            if (attrValue and attrValue.startswith('javascript:')) or attrName.startswith("on"):
                 attrValue = self._rewriteScript(attrValue)
 
             # special case: inline CSS/style attribute
@@ -189,11 +191,19 @@ class WBHtml(HTMLParser):
                     attrValue = self._rewriteMetaRefresh(attrValue)
 
             else:
+                # special case: base tag
+                if (tag == 'base') and (attrName == 'href') and attrValue:
+                    self.rewriter.setBaseUrl(attrValue)
+
                 rwMod = handler.get(attrName)
                 if rwMod is not None:
                     attrValue = self._rewriteURL(attrValue, rwMod)
 
-            self.out.write(' {0}="{1}"'.format(attrName, attrValue))
+            #self.out.write(' {0}="{1}"'.format(attrName, attrValue))
+            if attrValue:
+                self.out.write(' ' + attrName + '="' + attrValue + '"')
+            else:
+                self.out.write(' ' + attrName)
 
         self.out.write('/>' if isStartEnd else '>')