backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge branch 'master' into qa

Browse Source 
* master:
  send nice 503s and avoid scary stack traces...
This commit is contained in:
Noah Levitt 2018-10-26 15:27:02 -07:00
commit c19bbb73c2
3 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -40,7 +40,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.4b3.dev187', version='2.4b3.dev188',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

6
warcprox/controller.py
View File

@ -326,9 +326,9 @@ class WarcproxController(object):
to finish processing. to finish processing.
1. stop accepting new connections 1. stop accepting new connections
2. shut down active connections to remote servers (resulting in sending 2. shut down active connections to remote servers
http 502 to the proxy clients) 3. send "503 warcprox shutting down" response to active requests
3. shut down the postfetch processors one by one, in order, letting 4. shut down the postfetch processors one by one, in order, letting
them finish process their queues them finish process their queues
''' '''
with self._start_stop_lock: with self._start_stop_lock:

16
warcprox/mitmproxy.py
View File

@ -384,10 +384,16 @@ class MitmProxyHandler(http_server.BaseHTTPRequestHandler):
try: try:
return self._proxy_request() return self._proxy_request()
except Exception as e: except Exception as e:
self.logger.error( if self.server.shutting_down:
'error from remote server(?) %r: %r', self.logger.warn(
self.requestline, e, exc_info=True) 'sending 503 warcprox shutting down %r: %r',
self.send_error(502, str(e)) self.requestline, e)
self.send_error(503, 'warcprox shutting down')
else:
self.logger.error(
'error from remote server(?) %r: %r',
self.requestline, e, exc_info=True)
self.send_error(502, str(e))
return return
def send_error(self, code, message=None, explain=None): def send_error(self, code, message=None, explain=None):
@ -616,6 +622,7 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
PooledMixIn.__init__(self, options.max_threads) PooledMixIn.__init__(self, options.max_threads)
MitmProxy.__init__(self) MitmProxy.__init__(self)
self.profilers = collections.defaultdict(cProfile.Profile) self.profilers = collections.defaultdict(cProfile.Profile)
self.shutting_down = False
if options.profile: if options.profile:
self.process_request_thread = self._profile_process_request_thread self.process_request_thread = self._profile_process_request_thread
@ -648,6 +655,7 @@ class PooledMitmProxy(PooledMixIn, MitmProxy):
''' '''
Abort active connections to remote servers to achieve prompt shutdown. Abort active connections to remote servers to achieve prompt shutdown.
''' '''
self.shutting_down = True
for sock in self.remote_server_socks: for sock in self.remote_server_socks:
self.shutdown_request(sock) self.shutdown_request(sock)