backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

make --profile work again

Browse Source
This commit is contained in:
Noah Levitt 2018-01-16 15:58:29 -08:00
parent 6ff9030e67
commit 75486d0573
2 changed files with 16 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
warcprox/controller.py
View File

@ -398,18 +398,14 @@ class WarcproxController(object):
'aggregate performance profile of %s proxy threads:\n%s', 'aggregate performance profile of %s proxy threads:\n%s',
len(files), buf.getvalue()) len(files), buf.getvalue())
# warc writer threads # postfetch processors
files = [] for processor in self._postfetch_chain:
for wwt in self.warc_writer_threads: file = os.path.join(tmpdir, '%s.dat' % processor.ident)
file = os.path.join(tmpdir, '%s.dat' % wwt.ident) processor.profiler.dump_stats(file)
wwt.profiler.dump_stats(file) buf = io.StringIO()
files.append(file) stats = pstats.Stats(file, stream=buf)
stats.sort_stats('cumulative')
buf = io.StringIO() stats.print_stats(0.1)
stats = pstats.Stats(*files, stream=buf) self.logger.notice(
stats.sort_stats('cumulative') 'performance profile of %s:\n%s', processor,
stats.print_stats(0.1) buf.getvalue())
self.logger.notice(
'aggregate performance profile of %s warc writer threads:\n%s',
len(self.warc_writer_threads), buf.getvalue())

11
warcprox/mitmproxy.py
View File

@ -62,6 +62,8 @@ except ImportError:
import concurrent.futures import concurrent.futures
import urlcanon import urlcanon
import time import time
import collections
import cProfile
class ProxyingRecorder(object): class ProxyingRecorder(object):
""" """
@ -561,14 +563,14 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
# See also https://blog.dubbelboer.com/2012/04/09/syn-cookies.html # See also https://blog.dubbelboer.com/2012/04/09/syn-cookies.html
request_queue_size = 4096 request_queue_size = 4096
def __init__(self, max_threads, options=warcprox.Options()): def __init__(self, options=warcprox.Options()):
if options.max_threads: if options.max_threads:
self.logger.info( self.logger.info(
"max_threads=%s set by command line option", 'max_threads=%s set by command line option',
options.max_threads) options.max_threads)
PooledMixIn.__init__(self, options.max_threads) PooledMixIn.__init__(self, options.max_threads)
self.profilers = {} self.profilers = collections.defaultdict(cProfile.Profile)
if options.profile: if options.profile:
self.process_request_thread = self._profile_process_request_thread self.process_request_thread = self._profile_process_request_thread
@ -576,9 +578,6 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
self.process_request_thread = self._process_request_thread self.process_request_thread = self._process_request_thread
def _profile_process_request_thread(self, request, client_address): def _profile_process_request_thread(self, request, client_address):
if not threading.current_thread().ident in self.profilers:
import cProfile
self.profilers[threading.current_thread().ident] = cProfile.Profile()
profiler = self.profilers[threading.current_thread().ident] profiler = self.profilers[threading.current_thread().ident]
profiler.enable() profiler.enable()
self._process_request_thread(request, client_address) self._process_request_thread(request, client_address)