backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'trough-utf8' into qa

Browse Source 
* trough-utf8:
  make sure to send utf-8 to trough
  bump dev version after revert
  Revert "Merge pull request #67 from vbanos/update-ssl-ciphers"
  bump dev version number after PR merge
  Generate wildcard certs to reduce the number of certs generated
This commit is contained in:
Noah Levitt 2018-02-28 11:37:14 -08:00
commit 0841665b8d
4 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b2.dev150', version='2.4b2.dev152',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
warcprox/mitmproxy.py
View File

@ -64,7 +64,6 @@ import urlcanon
import time import time
import collections import collections
import cProfile import cProfile
from urllib3.util.ssl_ import DEFAULT_CIPHERS
class ProxyingRecorder(object): class ProxyingRecorder(object):
""" """
@ -260,7 +259,6 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
context = ssl.create_default_context() context = ssl.create_default_context()
context.check_hostname = False context.check_hostname = False
context.verify_mode = ssl.CERT_NONE context.verify_mode = ssl.CERT_NONE
context.ciphers = DEFAULT_CIPHERS
self._remote_server_sock = context.wrap_socket( self._remote_server_sock = context.wrap_socket(
self._remote_server_sock, server_hostname=self.hostname) self._remote_server_sock, server_hostname=self.hostname)
except AttributeError: except AttributeError:
@ -278,7 +276,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
return self._remote_server_sock return self._remote_server_sock
def _transition_to_ssl(self): def _transition_to_ssl(self):
certfile = self.server.ca.cert_for_host(self.hostname) certfile = self.server.ca.get_wildcard_cert(self.hostname)
self.request = self.connection = ssl.wrap_socket( self.request = self.connection = ssl.wrap_socket(
self.connection, server_side=True, certfile=certfile) self.connection, server_side=True, certfile=certfile)
# logging.info('self.hostname=%s certfile=%s', self.hostname, certfile) # logging.info('self.hostname=%s certfile=%s', self.hostname, certfile)

10
warcprox/trough.py
View File

@ -169,9 +169,12 @@ class TroughClient(object):
def write(self, segment_id, sql_tmpl, values=(), schema_id='default'): def write(self, segment_id, sql_tmpl, values=(), schema_id='default'):
write_url = self.write_url(segment_id, schema_id) write_url = self.write_url(segment_id, schema_id)
sql = sql_tmpl % tuple(self.sql_value(v) for v in values) sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
sql_bytes = sql.encode('utf-8')
try: try:
response = requests.post(write_url, sql, timeout=600) response = requests.post(
write_url, sql_bytes, timeout=600,
headers={'content-type': 'application/sql;charset=utf-8'})
if response.status_code != 200: if response.status_code != 200:
raise Exception( raise Exception(
'Received %s: %r in response to POST %s with data %r' % ( 'Received %s: %r in response to POST %s with data %r' % (
@ -199,8 +202,11 @@ class TroughClient(object):
if not read_url: if not read_url:
return None return None
sql = sql_tmpl % tuple(self.sql_value(v) for v in values) sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
sql_bytes = sql.encode('utf-8')
try: try:
response = requests.post(read_url, sql, timeout=600) response = requests.post(
read_url, sql_bytes, timeout=600,
headers={'content-type': 'application/sql;charset=utf-8'})
except: except:
self._read_url_cache.pop(segment_id, None) self._read_url_cache.pop(segment_id, None)
self.logger.error( self.logger.error(

5
warcprox/writer.py
View File

@ -37,11 +37,12 @@ except ImportError:
import contextlib import contextlib
class _OneWritableWarc: class _OneWritableWarc:
logger = logging.getLogger('warcprox.writer._OneWritableWarc')
''' '''
Utility class used by WarcWriter Utility class used by WarcWriter
''' '''
logger = logging.getLogger('warcprox.writer._OneWritableWarc')
def __init__(self, options=warcprox.Options(), randomtoken='0'): def __init__(self, options=warcprox.Options(), randomtoken='0'):
self.f = None self.f = None
self.path = None self.path = None