backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

bump up the socket backlog argument to try to stop kernel closing attempted connections on linux

Browse Source
This commit is contained in:
Noah Levitt 2017-05-05 18:49:56 +00:00
parent b2f08535ae
commit c642565ad8
2 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev75', version='2.1b1.dev76',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

16
warcprox/mitmproxy.py
View File

@ -462,9 +462,13 @@ class PooledMixIn(socketserver.ThreadingMixIn):
''' '''
# neither threading.Condition Queue.not_empty nor Queue.not_full do # neither threading.Condition Queue.not_empty nor Queue.not_full do
# what we need here, right? # what we need here, right?
self.logger.trace(
'someone is connecting qsize=%s', self.pool._work_queue.qsize())
while self.pool._work_queue.qsize() > 0: while self.pool._work_queue.qsize() > 0:
time.sleep(0.5) time.sleep(0.5)
return self.socket.accept() res = self.socket.accept()
self.logger.trace('accepted socket=%s', res)
return res
class MitmProxy(http_server.HTTPServer): class MitmProxy(http_server.HTTPServer):
def finish_request(self, request, client_address): def finish_request(self, request, client_address):
@ -492,6 +496,16 @@ class MitmProxy(http_server.HTTPServer):
self.shutdown_request(request) self.shutdown_request(request)
class PooledMitmProxy(PooledMixIn, MitmProxy): class PooledMitmProxy(PooledMixIn, MitmProxy):
# This value is passed as the "backlog" argument to listen(2). The default
# value from socketserver.TCPServer is 5. Increasing this value is part of
# the solution to client connections being closed suddenly and this message
# appearing in kernel log on linux: "TCP: request_sock_TCP: # Possible SYN
# flooding on port 8000. Sending cookies. Check SNMP # counters." I think
# this comes into play because we don't always accept(2) immediately (see
# PooledMixIn.get_request()).
# See also https://blog.dubbelboer.com/2012/04/09/syn-cookies.html
request_queue_size = 4096
def process_request_thread(self, request, client_address): def process_request_thread(self, request, client_address):
''' '''
This an almost verbatim copy/paste of This an almost verbatim copy/paste of