backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

aggregate warc writer thread profiles much like we do for proxy threads

Browse Source
This commit is contained in:
Noah Levitt 2017-11-14 16:44:31 -08:00
parent c13fd9a40e
commit 5c2c21de07
3 changed files with 25 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.2.1b2.dev117', version='2.2.1b2.dev118',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

24
warcprox/controller.py
View File

@ -44,7 +44,7 @@ class WarcproxController(object):
Create warcprox controller. Create warcprox controller.
If supplied, `proxy` should be an instance of WarcProxy, and If supplied, `proxy` should be an instance of WarcProxy, and
`warc_writer_threads` should be an list of WarcWriterThread instances. `warc_writer_threads` should be a list of WarcWriterThread instances.
If not supplied, they are created with default values. If not supplied, they are created with default values.
If supplied, playback_proxy should be an instance of PlaybackProxy. If If supplied, playback_proxy should be an instance of PlaybackProxy. If
@ -266,11 +266,9 @@ class WarcproxController(object):
self.shutdown() self.shutdown()
def _dump_profiling(self): def _dump_profiling(self):
import pstats import pstats, tempfile, os, io
import tempfile
import os
import io
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
# proxy threads
files = [] files = []
for th_id, profiler in self.proxy.profilers.items(): for th_id, profiler in self.proxy.profilers.items():
file = os.path.join(tmpdir, '%s.dat' % th_id) file = os.path.join(tmpdir, '%s.dat' % th_id)
@ -285,3 +283,19 @@ class WarcproxController(object):
'aggregate performance profile of %s proxy threads:\n%s', 'aggregate performance profile of %s proxy threads:\n%s',
len(files), buf.getvalue()) len(files), buf.getvalue())
# warc writer threads
files = []
for wwt in self.warc_writer_threads:
file = os.path.join(tmpdir, '%s.dat' % th_id)
profiler.dump_stats(file)
files.append(file)
buf = io.StringIO()
stats = pstats.Stats(*files, stream=buf)
stats.sort_stats('cumulative')
stats.print_stats(0.1)
self.logger.notice(
'aggregate performance profile of %s warc writer threads:\n%s',
len(self.warc_writer_threads), buf.getvalue())

18
warcprox/writerthread.py
View File

@ -56,23 +56,15 @@ class WarcWriterThread(threading.Thread):
self.idle = None self.idle = None
self.method_filter = set(method.upper() for method in self.options.method_filter or []) self.method_filter = set(method.upper() for method in self.options.method_filter or [])
def run(self):
if self.options.profile: if self.options.profile:
import cProfile import cProfile
import pstats self.profiler = cProfile.Profile()
import io
profiler = cProfile.Profile()
profiler.enable() def run(self):
if self.options.profile:
self.profiler.enable()
self._run() self._run()
profiler.disable() self.profiler.disable()
buf = io.StringIO()
stats = pstats.Stats(profiler, stream=buf)
stats.sort_stats('cumulative')
stats.print_stats(0.1)
self.logger.notice(
'%s performance profile:\n%s', self, buf.getvalue())
else: else:
self._run() self._run()