backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

change where RunningStats is initialized and fix tests

Browse Source
This commit is contained in:
Noah Levitt 2017-12-29 11:06:46 -08:00
parent c966f7f6e8
commit 5347cc92c3
4 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -52,7 +52,7 @@ except:
setuptools.setup(
name='warcprox',
version='2.3.1b4.dev133',
version='2.3.1b4.dev134',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',

5
tests/test_warcprox.py
View File

@ -1308,7 +1308,9 @@ def test_status_api(warcprox_):
status = json.loads(response.content.decode('ascii'))
assert set(status.keys()) == {
'role', 'version', 'host', 'address', 'port', 'pid', 'load',
'queued_urls', 'queue_max_size', 'seconds_behind', 'threads'}
'queued_urls', 'queue_max_size', 'seconds_behind', 'threads',
'rates_5min', 'rates_1min', 'unaccepted_requests', 'rates_15min',
'active_requests',}
assert status['role'] == 'warcprox'
assert status['version'] == warcprox.__version__
assert status['port'] == warcprox_.proxy.server_port
@ -1361,6 +1363,7 @@ def test_controller_with_defaults():
assert controller.proxy.recorded_url_q
assert controller.proxy.server_address == ('127.0.0.1', 8000)
assert controller.proxy.server_port == 8000
assert controller.proxy.running_stats
for wwt in controller.warc_writer_threads:
assert wwt
assert wwt.recorded_url_q

2
warcprox/controller.py
View File

@ -62,7 +62,7 @@ class WarcproxController(object):
warcprox.writerthread.WarcWriterThread(
name='WarcWriterThread%03d' % i,
recorded_url_q=self.proxy.recorded_url_q,
options=options)
listeners=[self.proxy.running_stats], options=options)
for i in range(int(self.proxy.max_threads ** 0.5))]
self.proxy_thread = None

7
warcprox/warcproxy.py
View File

@ -376,7 +376,7 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
def __init__(
self, ca=None, recorded_url_q=None, stats_db=None,
running_stats=None, options=warcprox.Options()):
options=warcprox.Options()):
server_address = (
options.address or 'localhost',
options.port if options.port is not None else 8000)
@ -410,9 +410,10 @@ class SingleThreadedWarcProxy(http_server.HTTPServer, object):
maxsize=options.queue_size or 1000)
self.stats_db = stats_db
self.running_stats = running_stats
self.options = options
self.running_stats = warcprox.stats.RunningStats()
def status(self):
if hasattr(super(), 'status'):
result = super().status()
@ -458,7 +459,7 @@ class WarcProxy(SingleThreadedWarcProxy, warcprox.mitmproxy.PooledMitmProxy):
warcprox.mitmproxy.PooledMitmProxy.__init__(
self, options.max_threads, options)
SingleThreadedWarcProxy.__init__(
self, ca, recorded_url_q, stats_db, running_stats, options)
self, ca, recorded_url_q, stats_db, options)
def server_activate(self):
http_server.HTTPServer.server_activate(self)