diff --git a/warcprox/controller.py b/warcprox/controller.py index efa7885..dabbf6a 100644 --- a/warcprox/controller.py +++ b/warcprox/controller.py @@ -398,18 +398,14 @@ class WarcproxController(object): 'aggregate performance profile of %s proxy threads:\n%s', len(files), buf.getvalue()) - # warc writer threads - files = [] - for wwt in self.warc_writer_threads: - file = os.path.join(tmpdir, '%s.dat' % wwt.ident) - wwt.profiler.dump_stats(file) - files.append(file) - - buf = io.StringIO() - stats = pstats.Stats(*files, stream=buf) - stats.sort_stats('cumulative') - stats.print_stats(0.1) - self.logger.notice( - 'aggregate performance profile of %s warc writer threads:\n%s', - len(self.warc_writer_threads), buf.getvalue()) - + # postfetch processors + for processor in self._postfetch_chain: + file = os.path.join(tmpdir, '%s.dat' % processor.ident) + processor.profiler.dump_stats(file) + buf = io.StringIO() + stats = pstats.Stats(file, stream=buf) + stats.sort_stats('cumulative') + stats.print_stats(0.1) + self.logger.notice( + 'performance profile of %s:\n%s', processor, + buf.getvalue()) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index c42fb68..608cc75 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -62,6 +62,8 @@ except ImportError: import concurrent.futures import urlcanon import time +import collections +import cProfile class ProxyingRecorder(object): """ @@ -561,14 +563,14 @@ class PooledMitmProxy(PooledMixIn, MitmProxy): # See also https://blog.dubbelboer.com/2012/04/09/syn-cookies.html request_queue_size = 4096 - def __init__(self, max_threads, options=warcprox.Options()): + def __init__(self, options=warcprox.Options()): if options.max_threads: self.logger.info( - "max_threads=%s set by command line option", + 'max_threads=%s set by command line option', options.max_threads) PooledMixIn.__init__(self, options.max_threads) - self.profilers = {} + self.profilers = collections.defaultdict(cProfile.Profile) if options.profile: self.process_request_thread = self._profile_process_request_thread @@ -576,9 +578,6 @@ class PooledMitmProxy(PooledMixIn, MitmProxy): self.process_request_thread = self._process_request_thread def _profile_process_request_thread(self, request, client_address): - if not threading.current_thread().ident in self.profilers: - import cProfile - self.profilers[threading.current_thread().ident] = cProfile.Profile() profiler = self.profilers[threading.current_thread().ident] profiler.enable() self._process_request_thread(request, client_address)