diff --git a/warcprox/main.py b/warcprox/main.py index f13d2dd..358611d 100644 --- a/warcprox/main.py +++ b/warcprox/main.py @@ -162,9 +162,9 @@ def _build_arg_parser(prog='warcprox'): default=None, help=( 'host:port of tor socks proxy, used only to connect to ' '.onion sites')) - # Configurable connection timeout to target sites, default is 60 sec. + # Configurable connection socket timeout, default is 60 sec. arg_parser.add_argument( - '--remote-server-timeout', dest='remote_server_timeout', type=float, + '--socket-timeout', dest='socket_timeout', type=float, default=None, help=argparse.SUPPRESS) arg_parser.add_argument( '--crawl-log-dir', dest='crawl_log_dir', default=None, help=( diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index b25901b..95d5b31 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -205,13 +205,13 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): and records the bytes in transit as it proxies them. ''' logger = logging.getLogger("warcprox.mitmproxy.MitmProxyHandler") - _remote_server_timeout = 60 + _socket_timeout = 60 def __init__(self, request, client_address, server): threading.current_thread().name = 'MitmProxyHandler(tid={},started={},client={}:{})'.format(warcprox.gettid(), datetime.datetime.utcnow().isoformat(), client_address[0], client_address[1]) self.is_connect = False self._headers_buffer = [] - request.settimeout(60) # XXX what value should this have? + request.settimeout(self._socket_timeout) http_server.BaseHTTPRequestHandler.__init__(self, request, client_address, server) def _determine_host_port(self): @@ -248,8 +248,7 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler): self._remote_server_sock = socket.socket() self._remote_server_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - # XXX what value should this timeout have? - self._remote_server_sock.settimeout(self._remote_server_timeout) + self._remote_server_sock.settimeout(self._socket_timeout) self._remote_server_sock.connect((self.hostname, int(self.port))) # Wrap socket if SSL is required diff --git a/warcprox/warcproxy.py b/warcprox/warcproxy.py index 6d8accb..5b36300 100644 --- a/warcprox/warcproxy.py +++ b/warcprox/warcproxy.py @@ -397,8 +397,8 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object): WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy WarcProxyHandler.onion_tor_socks_proxy_port = None - if options.remote_server_timeout: - WarcProxyHandler._remote_server_timeout = options.remote_server_timeout + if options.socket_timeout: + WarcProxyHandler._socket_timeout = options.socket_timeout http_server.HTTPServer.__init__( self, server_address, WarcProxyHandler, bind_and_activate=True)