backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

try to fix https://github.com/internetarchive/warcprox/issues/27

Browse Source
This commit is contained in:
Noah Levitt 2017-07-07 14:54:55 -07:00
parent 5a8d1610e6
commit b1a8fecd9d
2 changed files with 18 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -50,7 +50,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev92', version='2.1b1.dev93',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

27
warcprox/mitmproxy.py
View File

@ -52,7 +52,6 @@ try:
import socketserver import socketserver
except ImportError: except ImportError:
import SocketServer as socketserver import SocketServer as socketserver
import resource
import concurrent.futures import concurrent.futures
import urlcanon import urlcanon
import time import time
@ -440,15 +439,23 @@ class PooledMixIn(socketserver.ThreadingMixIn):
# man getrlimit: "RLIMIT_NPROC The maximum number of processes (or, # man getrlimit: "RLIMIT_NPROC The maximum number of processes (or,
# more precisely on Linux, threads) that can be created for the # more precisely on Linux, threads) that can be created for the
# real user ID of the calling process." # real user ID of the calling process."
rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0] try:
rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0] import resource
max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2) rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0]
# resource.RLIM_INFINITY == -1 which can result in max_threads == 0 rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
if max_threads <= 0 or max_threads > 5000: max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2)
max_threads = 5000 # resource.RLIM_INFINITY == -1 which can result in max_threads == 0
self.logger.info( if max_threads <= 0 or max_threads > 5000:
"max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)", max_threads = 5000
max_threads, rlimit_nproc, rlimit_nofile) self.logger.info(
"max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)",
max_threads, rlimit_nproc, rlimit_nofile)
except Exception as e:
self.logger.warn(
"unable to calculate optimal number of threads based "
"on resource limits due to %s", e)
max_threads = 100
self.logger.info("max_threads=%s", max_threads)
self.max_threads = max_threads self.max_threads = max_threads
self.pool = concurrent.futures.ThreadPoolExecutor(max_threads) self.pool = concurrent.futures.ThreadPoolExecutor(max_threads)