backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

added custom requests HTTPAdapter, PywbHttpAdapter, that restores the behavior of urllib3 < 1.25.x which was to not verify ssl certs fixes #467 (#469)

Browse Source
This commit is contained in:
John Berlin 2019-05-28 16:12:00 -04:00 committed by Ilya Kreymer
parent da3d11a44a
commit 612ead32f8
1 changed files with 37 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
pywb/warcserver/http.py
View File

@ -1,27 +1,53 @@
from requests.adapters import HTTPAdapter
import requests
import os import os
import requests
import six.moves.http_client import six.moves.http_client
from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter
from urllib3.poolmanager import PoolManager
six.moves.http_client._MAXHEADERS = 10000 six.moves.http_client._MAXHEADERS = 10000
six.moves.http_client._MAXLINE = 131072
SOCKS_PROXIES = None SOCKS_PROXIES = None
orig_getaddrinfo = None orig_getaddrinfo = None
#============================================================================= class PywbHttpAdapter(HTTPAdapter):
"""This adaptor exists exists to restore the default behavior
of urllib3 < 1.25.x, which was to not verify ssl certs,
until a better solution is found
"""
def init_poolmanager(
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
cert_reqs='CERT_NONE',
**pool_kwargs
)
# =============================================================================
class DefaultAdapters(object): class DefaultAdapters(object):
live_adapter = HTTPAdapter(max_retries=3) live_adapter = PywbHttpAdapter(max_retries=3)
remote_adapter = HTTPAdapter(max_retries=3) remote_adapter = PywbHttpAdapter(max_retries=3)
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
#============================================================================= # =============================================================================
def patch_socks(): def patch_socks():
try: try:
import socks import socks
except ImportError: #pragma: no cover except ImportError: # pragma: no cover
print('Ignoring SOCKS_HOST: PySocks must be installed to use SOCKS proxy') print('Ignoring SOCKS_HOST: PySocks must be installed to use SOCKS proxy')
return return
@ -32,7 +58,7 @@ def patch_socks():
# Set socks proxy and wrap the urllib module # Set socks proxy and wrap the urllib module
socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, socks_host, socks_port, True) socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, socks_host, socks_port, True)
#socket.socket = socks.socksocket # sets default socket to be the sockipy socket # socket.socket = socks.socksocket # sets default socket to be the sockipy socket
# store original getaddrinfo # store original getaddrinfo
global orig_getaddrinfo global orig_getaddrinfo
@ -53,8 +79,8 @@ def patch_socks():
socks_url = 'socks5h://{0}:{1}'.format(socks_host, socks_port) socks_url = 'socks5h://{0}:{1}'.format(socks_host, socks_port)
global SOCKS_PROXIES global SOCKS_PROXIES
SOCKS_PROXIES = {'http': socks_url, SOCKS_PROXIES = {'http': socks_url, 'https': socks_url}
'https': socks_url}
# ============================================================================= # =============================================================================
def unpatch_socks(): def unpatch_socks():
@ -63,6 +89,7 @@ def unpatch_socks():
return return
import socks import socks
socks.socket.getaddrinfo = orig_getaddrinfo socks.socket.getaddrinfo = orig_getaddrinfo
orig_getaddrinfo = None orig_getaddrinfo = None
@ -73,6 +100,3 @@ def unpatch_socks():
# ============================================================================= # =============================================================================
if os.environ.get('SOCKS_HOST'): if os.environ.get('SOCKS_HOST'):
patch_socks() patch_socks()