backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

make --profile work again

Browse Source
This commit is contained in:
Noah Levitt 2018-01-16 15:58:29 -08:00
parent 6ff9030e67
commit 75486d0573
2 changed files with 16 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
warcprox/controller.py
View File

@ -398,18 +398,14 @@ class WarcproxController(object):
'aggregate performance profile of %s proxy threads:\n%s',
len(files), buf.getvalue())
# warc writer threads
files = []
for wwt in self.warc_writer_threads:
file = os.path.join(tmpdir, '%s.dat' % wwt.ident)
wwt.profiler.dump_stats(file)
files.append(file)
buf = io.StringIO()
stats = pstats.Stats(*files, stream=buf)
stats.sort_stats('cumulative')
stats.print_stats(0.1)
self.logger.notice(
'aggregate performance profile of %s warc writer threads:\n%s',
len(self.warc_writer_threads), buf.getvalue())
# postfetch processors
for processor in self._postfetch_chain:
file = os.path.join(tmpdir, '%s.dat' % processor.ident)
processor.profiler.dump_stats(file)
buf = io.StringIO()
stats = pstats.Stats(file, stream=buf)
stats.sort_stats('cumulative')
stats.print_stats(0.1)
self.logger.notice(
'performance profile of %s:\n%s', processor,
buf.getvalue())

11
warcprox/mitmproxy.py
View File

@ -62,6 +62,8 @@ except ImportError:
import concurrent.futures
import urlcanon
import time
import collections
import cProfile
class ProxyingRecorder(object):
"""
@ -561,14 +563,14 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
# See also https://blog.dubbelboer.com/2012/04/09/syn-cookies.html
request_queue_size = 4096
def __init__(self, max_threads, options=warcprox.Options()):
def __init__(self, options=warcprox.Options()):
if options.max_threads:
self.logger.info(
"max_threads=%s set by command line option",
'max_threads=%s set by command line option',
options.max_threads)
PooledMixIn.__init__(self, options.max_threads)
self.profilers = {}
self.profilers = collections.defaultdict(cProfile.Profile)
if options.profile:
self.process_request_thread = self._profile_process_request_thread
@ -576,9 +578,6 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
self.process_request_thread = self._process_request_thread
def _profile_process_request_thread(self, request, client_address):
if not threading.current_thread().ident in self.profilers:
import cProfile
self.profilers[threading.current_thread().ident] = cProfile.Profile()
profiler = self.profilers[threading.current_thread().ident]
profiler.enable()
self._process_request_thread(request, client_address)