diff --git a/setup.py b/setup.py index 31ee69f..c4b80e1 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ except: setuptools.setup( name='warcprox', - version='2.4b2.dev150', + version='2.4b2.dev152', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index f0f8a77..1bbd930 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -64,7 +64,6 @@ import urlcanon import time import collections import cProfile -from urllib3.util.ssl_ import DEFAULT_CIPHERS class ProxyingRecorder(object): """ @@ -260,7 +259,6 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE - context.ciphers = DEFAULT_CIPHERS self._remote_server_sock = context.wrap_socket( self._remote_server_sock, server_hostname=self.hostname) except AttributeError: @@ -278,7 +276,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): return self._remote_server_sock def _transition_to_ssl(self): - certfile = self.server.ca.cert_for_host(self.hostname) + certfile = self.server.ca.get_wildcard_cert(self.hostname) self.request = self.connection = ssl.wrap_socket( self.connection, server_side=True, certfile=certfile) # logging.info('self.hostname=%s certfile=%s', self.hostname, certfile) diff --git a/warcprox/trough.py b/warcprox/trough.py index 4128e50..b7db127 100644 --- a/warcprox/trough.py +++ b/warcprox/trough.py @@ -169,9 +169,12 @@ class TroughClient(object): def write(self, segment_id, sql_tmpl, values=(), schema_id='default'): write_url = self.write_url(segment_id, schema_id) sql = sql_tmpl % tuple(self.sql_value(v) for v in values) + sql_bytes = sql.encode('utf-8') try: - response = requests.post(write_url, sql, timeout=600) + response = requests.post( + write_url, sql_bytes, timeout=600, + headers={'content-type': 'application/sql;charset=utf-8'}) if response.status_code != 200: raise Exception( 'Received %s: %r in response to POST %s with data %r' % ( @@ -199,8 +202,11 @@ class TroughClient(object): if not read_url: return None sql = sql_tmpl % tuple(self.sql_value(v) for v in values) + sql_bytes = sql.encode('utf-8') try: - response = requests.post(read_url, sql, timeout=600) + response = requests.post( + read_url, sql_bytes, timeout=600, + headers={'content-type': 'application/sql;charset=utf-8'}) except: self._read_url_cache.pop(segment_id, None) self.logger.error( diff --git a/warcprox/writer.py b/warcprox/writer.py index dbabafb..e8f4b79 100644 --- a/warcprox/writer.py +++ b/warcprox/writer.py @@ -37,11 +37,12 @@ except ImportError: import contextlib class _OneWritableWarc: - logger = logging.getLogger('warcprox.writer._OneWritableWarc') - ''' Utility class used by WarcWriter ''' + + logger = logging.getLogger('warcprox.writer._OneWritableWarc') + def __init__(self, options=warcprox.Options(), randomtoken='0'): self.f = None self.path = None