Merge branch 'trough-utf8' into qa

* trough-utf8:
  make sure to send utf-8 to trough
  bump dev version after revert
  Revert "Merge pull request #67 from vbanos/update-ssl-ciphers"
  bump dev version number after PR merge
  Generate wildcard certs to reduce the number of certs generated
This commit is contained in:
Noah Levitt 2018-02-28 11:37:14 -08:00
commit 0841665b8d
4 changed files with 13 additions and 8 deletions

View File

@ -40,7 +40,7 @@ except:
setuptools.setup(
name='warcprox',
version='2.4b2.dev150',
version='2.4b2.dev152',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',

View File

@ -64,7 +64,6 @@ import urlcanon
import time
import collections
import cProfile
from urllib3.util.ssl_ import DEFAULT_CIPHERS
class ProxyingRecorder(object):
"""
@ -260,7 +259,6 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
context.ciphers = DEFAULT_CIPHERS
self._remote_server_sock = context.wrap_socket(
self._remote_server_sock, server_hostname=self.hostname)
except AttributeError:
@ -278,7 +276,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
return self._remote_server_sock
def _transition_to_ssl(self):
certfile = self.server.ca.cert_for_host(self.hostname)
certfile = self.server.ca.get_wildcard_cert(self.hostname)
self.request = self.connection = ssl.wrap_socket(
self.connection, server_side=True, certfile=certfile)
# logging.info('self.hostname=%s certfile=%s', self.hostname, certfile)

View File

@ -169,9 +169,12 @@ class TroughClient(object):
def write(self, segment_id, sql_tmpl, values=(), schema_id='default'):
write_url = self.write_url(segment_id, schema_id)
sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
sql_bytes = sql.encode('utf-8')
try:
response = requests.post(write_url, sql, timeout=600)
response = requests.post(
write_url, sql_bytes, timeout=600,
headers={'content-type': 'application/sql;charset=utf-8'})
if response.status_code != 200:
raise Exception(
'Received %s: %r in response to POST %s with data %r' % (
@ -199,8 +202,11 @@ class TroughClient(object):
if not read_url:
return None
sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
sql_bytes = sql.encode('utf-8')
try:
response = requests.post(read_url, sql, timeout=600)
response = requests.post(
read_url, sql_bytes, timeout=600,
headers={'content-type': 'application/sql;charset=utf-8'})
except:
self._read_url_cache.pop(segment_id, None)
self.logger.error(

View File

@ -37,11 +37,12 @@ except ImportError:
import contextlib
class _OneWritableWarc:
logger = logging.getLogger('warcprox.writer._OneWritableWarc')
'''
Utility class used by WarcWriter
'''
logger = logging.getLogger('warcprox.writer._OneWritableWarc')
def __init__(self, options=warcprox.Options(), randomtoken='0'):
self.f = None
self.path = None