backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

add another field to status api and service registry, "threads", the size of the proxy server thread pool

Browse Source
This commit is contained in:
Noah Levitt 2017-03-30 16:18:50 -07:00
parent 35d7ccd12e
commit f17584836e
4 changed files with 22 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.1b1.dev64', version='2.1b1.dev65',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

35
tests/test_warcprox.py
View File

@ -1285,32 +1285,35 @@ def test_status_api(warcprox_):
url = 'http://localhost:%s/status' % warcprox_.proxy.server_port url = 'http://localhost:%s/status' % warcprox_.proxy.server_port
response = requests.get(url) response = requests.get(url)
assert response.status_code == 200 assert response.status_code == 200
response_dict = json.loads(response.content.decode('ascii')) status = json.loads(response.content.decode('ascii'))
assert set(response_dict.keys()) == { assert set(status.keys()) == {
'role', 'version', 'host', 'address', 'port', 'pid', 'load', 'role', 'version', 'host', 'address', 'port', 'pid', 'load',
'queued_urls', 'queue_max_size', 'seconds_behind'} 'queued_urls', 'queue_max_size', 'seconds_behind', 'threads'}
assert response_dict['role'] == 'warcprox' assert status['role'] == 'warcprox'
assert response_dict['version'] == warcprox.__version__ assert status['version'] == warcprox.__version__
assert response_dict['port'] == warcprox_.proxy.server_port assert status['port'] == warcprox_.proxy.server_port
assert response_dict['pid'] == os.getpid() assert status['pid'] == os.getpid()
assert status['threads'] == warcprox_.proxy.pool._max_workers
def test_svcreg_status(warcprox_, service_registry): def test_svcreg_status(warcprox_, service_registry):
if service_registry: if service_registry:
start = time.time() start = time.time()
while time.time() - start < 15: while time.time() - start < 15:
svc = service_registry.available_service('warcprox') status = service_registry.available_service('warcprox')
if svc: if status:
break break
time.sleep(0.5) time.sleep(0.5)
assert svc assert status
assert set(svc.keys()) == { assert set(status.keys()) == {
'id', 'role', 'version', 'host', 'port', 'pid', 'load', 'id', 'role', 'version', 'host', 'port', 'pid', 'load',
'queued_urls', 'queue_max_size', 'seconds_behind', 'queued_urls', 'queue_max_size', 'seconds_behind',
'first_heartbeat', 'heartbeat_interval', 'last_heartbeat'} 'first_heartbeat', 'heartbeat_interval', 'last_heartbeat',
assert svc['role'] == 'warcprox' 'threads'}
assert svc['version'] == warcprox.__version__ assert status['role'] == 'warcprox'
assert svc['port'] == warcprox_.proxy.server_port assert status['version'] == warcprox.__version__
assert svc['pid'] == os.getpid() assert status['port'] == warcprox_.proxy.server_port
assert status['pid'] == os.getpid()
assert status['threads'] == warcprox_.proxy.pool._max_workers
def test_timestamped_queue(): def test_timestamped_queue():
# see also test_queue.py # see also test_queue.py

1
warcprox/controller.py
View File

@ -149,6 +149,7 @@ class WarcproxController(object):
status_info['queued_urls'] = self.proxy.recorded_url_q.qsize() status_info['queued_urls'] = self.proxy.recorded_url_q.qsize()
status_info['queue_max_size'] = self.proxy.recorded_url_q.maxsize status_info['queue_max_size'] = self.proxy.recorded_url_q.maxsize
status_info['seconds_behind'] = self.proxy.recorded_url_q.seconds_behind() status_info['seconds_behind'] = self.proxy.recorded_url_q.seconds_behind()
status_info['threads'] = self.proxy.pool._max_workers
self.status_info = self.service_registry.heartbeat(status_info) self.status_info = self.service_registry.heartbeat(status_info)
self.logger.log( self.logger.log(

1
warcprox/warcproxy.py
View File

@ -213,6 +213,7 @@ class WarcProxyHandler(warcprox.mitmproxy.MitmProxyHandler):
'queue_max_size': self.server.recorded_url_q.maxsize, 'queue_max_size': self.server.recorded_url_q.maxsize,
'seconds_behind': self.server.recorded_url_q.seconds_behind(), 'seconds_behind': self.server.recorded_url_q.seconds_behind(),
'pid': os.getpid(), 'pid': os.getpid(),
'threads': self.server.pool._max_workers,
} }
payload = json.dumps( payload = json.dumps(
status_info, indent=2).encode('utf-8') + b'\n' status_info, indent=2).encode('utf-8') + b'\n'