From b1a8fecd9d15dc58bbc05b46ae7e369304cb019a Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 7 Jul 2017 14:54:55 -0700 Subject: [PATCH] try to fix https://github.com/internetarchive/warcprox/issues/27 --- setup.py | 2 +- warcprox/mitmproxy.py | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index c06fed7..5c159a0 100755 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ except: setuptools.setup( name='warcprox', - version='2.1b1.dev92', + version='2.1b1.dev93', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index ec9dafc..6297dcc 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -52,7 +52,6 @@ try: import socketserver except ImportError: import SocketServer as socketserver -import resource import concurrent.futures import urlcanon import time @@ -440,15 +439,23 @@ class PooledMixIn(socketserver.ThreadingMixIn): # man getrlimit: "RLIMIT_NPROC The maximum number of processes (or, # more precisely on Linux, threads) that can be created for the # real user ID of the calling process." - rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0] - rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2) - # resource.RLIM_INFINITY == -1 which can result in max_threads == 0 - if max_threads <= 0 or max_threads > 5000: - max_threads = 5000 - self.logger.info( - "max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)", - max_threads, rlimit_nproc, rlimit_nofile) + try: + import resource + rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0] + rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2) + # resource.RLIM_INFINITY == -1 which can result in max_threads == 0 + if max_threads <= 0 or max_threads > 5000: + max_threads = 5000 + self.logger.info( + "max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)", + max_threads, rlimit_nproc, rlimit_nofile) + except Exception as e: + self.logger.warn( + "unable to calculate optimal number of threads based " + "on resource limits due to %s", e) + max_threads = 100 + self.logger.info("max_threads=%s", max_threads) self.max_threads = max_threads self.pool = concurrent.futures.ThreadPoolExecutor(max_threads)