added custom requests HTTPAdapter, PywbHttpAdapter, that restores the behavior of urllib3 < 1.25.x which was to not verify ssl certs fixes #467 (#469)

pywb/warcserver/http.py

@@ -1,27 +1,53 @@
-from requests.adapters import HTTPAdapter
-import requests
 import os
 
+import requests
 import six.moves.http_client
+from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter
+from urllib3.poolmanager import PoolManager
+
 six.moves.http_client._MAXHEADERS = 10000
+six.moves.http_client._MAXLINE = 131072
 
 SOCKS_PROXIES = None
 orig_getaddrinfo = None
 
 
-#=============================================================================
+class PywbHttpAdapter(HTTPAdapter):
+    """This adaptor exists exists to restore the default behavior
+    of urllib3 < 1.25.x, which was to not verify ssl certs,
+    until a better solution is found
+    """
+
+    def init_poolmanager(
+        self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
+    ):
+        self._pool_connections = connections
+        self._pool_maxsize = maxsize
+        self._pool_block = block
+        self.poolmanager = PoolManager(
+            num_pools=connections,
+            maxsize=maxsize,
+            block=block,
+            strict=True,
+            cert_reqs='CERT_NONE',
+            **pool_kwargs
+        )
+
+
+# =============================================================================
 class DefaultAdapters(object):
-    live_adapter = HTTPAdapter(max_retries=3)
-    remote_adapter = HTTPAdapter(max_retries=3)
+    live_adapter = PywbHttpAdapter(max_retries=3)
+    remote_adapter = PywbHttpAdapter(max_retries=3)
+
 
 requests.packages.urllib3.disable_warnings()
 
 
-#=============================================================================
+# =============================================================================
 def patch_socks():
     try:
         import socks
-    except ImportError:  #pragma: no cover
+    except ImportError:  # pragma: no cover
         print('Ignoring SOCKS_HOST: PySocks must be installed to use SOCKS proxy')
         return
 
@@ -32,7 +58,7 @@ def patch_socks():
 
     # Set socks proxy and wrap the urllib module
     socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, socks_host, socks_port, True)
-    #socket.socket = socks.socksocket # sets default socket to be the sockipy socket
+    # socket.socket = socks.socksocket # sets default socket to be the sockipy socket
 
     # store original getaddrinfo
     global orig_getaddrinfo
@@ -53,8 +79,8 @@ def patch_socks():
     socks_url = 'socks5h://{0}:{1}'.format(socks_host, socks_port)
 
     global SOCKS_PROXIES
-    SOCKS_PROXIES = {'http': socks_url,
-                     'https': socks_url}
+    SOCKS_PROXIES = {'http': socks_url, 'https': socks_url}
+
 
 # =============================================================================
 def unpatch_socks():
@@ -63,6 +89,7 @@ def unpatch_socks():
         return
 
     import socks
+
     socks.socket.getaddrinfo = orig_getaddrinfo
     orig_getaddrinfo = None
 
@@ -73,6 +100,3 @@ def unpatch_socks():
 # =============================================================================
 if os.environ.get('SOCKS_HOST'):
     patch_socks()
-
-
-