bufferedreader: support brotli decompression

rewrite: handle Content-Encoding: br using brotli decompressor
setup: add brotlipy as dependency

pywb/rewrite/rewrite_content.py

@@ -77,6 +77,7 @@ class RewriteContent(object):
 
 
     def _check_encoding(self, rewritten_headers, stream, enc):
+        matched = False
         if (rewritten_headers.
              contains_removed_header('content-encoding', enc)):
 
@@ -87,8 +88,9 @@ class RewriteContent(object):
                 stream = DecompressingBufferedReader(stream, decomp_type=enc)
 
             rewritten_headers.status_headers.remove_header('content-length')
+            matched = True
 
-        return stream
+        return matched, stream
 
 
 
@@ -139,8 +141,12 @@ class RewriteContent(object):
         encoding = None
         first_buff = b''
 
-        stream = self._check_encoding(rewritten_headers, stream, 'gzip')
-        stream = self._check_encoding(rewritten_headers, stream, 'deflate')
+        for decomp_type in BufferedReader.get_supported_decompressors():
+            matched, stream = self._check_encoding(rewritten_headers,
+                                                   stream,
+                                                   decomp_type)
+            if matched:
+                break
 
         if mod == 'js_':
             text_type, stream = self._resolve_text_type('js',

pywb/utils/bufferedreaders.py

@@ -1,5 +1,6 @@
 from io import BytesIO
 import zlib
+import brotli
 
 
 #=================================================================
@@ -17,6 +18,11 @@ def deflate_decompressor():
 def deflate_decompressor_alt():
     return zlib.decompressobj(-zlib.MAX_WBITS)
 
+def brotli_decompressor():
+    decomp = brotli.Decompressor()
+    decomp.unused_data = None
+    return decomp
+
 
 #=================================================================
 class BufferedReader(object):
@@ -40,7 +46,9 @@ class BufferedReader(object):
 
     DECOMPRESSORS = {'gzip': gzip_decompressor,
                      'deflate': deflate_decompressor,
-                     'deflate_alt': deflate_decompressor_alt}
+                     'deflate_alt': deflate_decompressor_alt,
+                     'br': brotli_decompressor
+                    }
 
     def __init__(self, stream, block_size=1024,
                  decomp_type=None,
@@ -181,6 +189,10 @@ class BufferedReader(object):
             self.stream.close()
             self.stream = None
 
+    @classmethod
+    def get_supported_decompressors(cls):
+        return cls.DECOMPRESSORS.keys()
+
 
 #=================================================================
 class DecompressingBufferedReader(BufferedReader):

setup.py

@@ -79,6 +79,7 @@ setup(
         'redis',
         'jinja2',
         'surt>=0.3.0',
+        'brotlipy',
         'pyyaml',
         'watchdog',
         'webencodings',