backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Rename remote-server-timeout to socket-timeout

Browse Source 
Also apply it to both remote target and local proxy client connections.
This commit is contained in:
Vangelis Banos 2018-01-30 07:03:58 +00:00 committed by Noah Levitt
parent 428a03689f
commit 9474a7ae7f
3 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/main.py
View File

@ -162,9 +162,9 @@ def _build_arg_parser(prog='warcprox'):
default=None, help=(
'host:port of tor socks proxy, used only to connect to '
'.onion sites'))
# Configurable connection timeout to target sites, default is 60 sec.
# Configurable connection socket timeout, default is 60 sec.
arg_parser.add_argument(
'--remote-server-timeout', dest='remote_server_timeout', type=float,
'--socket-timeout', dest='socket_timeout', type=float,
default=None, help=argparse.SUPPRESS)
arg_parser.add_argument(
'--crawl-log-dir', dest='crawl_log_dir', default=None, help=(

7
warcprox/mitmproxy.py
View File

@ -205,13 +205,13 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
and records the bytes in transit as it proxies them.
'''
logger = logging.getLogger("warcprox.mitmproxy.MitmProxyHandler")
_remote_server_timeout = 60
_socket_timeout = 60
def __init__(self, request, client_address, server):
threading.current_thread().name = 'MitmProxyHandler(tid={},started={},client={}:{})'.format(warcprox.gettid(), datetime.datetime.utcnow().isoformat(), client_address[0], client_address[1])
self.is_connect = False
self._headers_buffer = []
request.settimeout(60) # XXX what value should this have?
request.settimeout(self._socket_timeout)
http_server.BaseHTTPRequestHandler.__init__(self, request, client_address, server)
def _determine_host_port(self):
@ -248,8 +248,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
self._remote_server_sock = socket.socket()
self._remote_server_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# XXX what value should this timeout have?
self._remote_server_sock.settimeout(self._remote_server_timeout)
self._remote_server_sock.settimeout(self._socket_timeout)
self._remote_server_sock.connect((self.hostname, int(self.port)))
# Wrap socket if SSL is required

4
warcprox/warcproxy.py
View File

@ -397,8 +397,8 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy
WarcProxyHandler.onion_tor_socks_proxy_port = None
if options.remote_server_timeout:
WarcProxyHandler._remote_server_timeout = options.remote_server_timeout
if options.socket_timeout:
WarcProxyHandler._socket_timeout = options.socket_timeout
http_server.HTTPServer.__init__(
self, server_address, WarcProxyHandler, bind_and_activate=True)