mirror of
https://github.com/internetarchive/warcprox.git
synced 2025-01-18 13:22:09 +01:00
Handle RuntimeError
Some times when warcprox runs for several days under load it freezes and the last error in the log is: ``` WARNING:warcprox.warcproxy.WarcProxy:exception processing request <socket.socket fd=53, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('207.241.225.241', 8003), raddr=('207.241.225.241', 40738)> from ('207.241.225.241', 40738) Traceback (most recent call last): File "/usr/lib/python3.7/socketserver.py", line 316, in _handle_request_noblock self.process_request(request, client_address) File "/opt/spn2/lib/python3.7/site-packages/warcprox/mitmproxy.py", line 641, in process_request self.process_request_thread, request, client_address) File "/usr/lib/python3.7/concurrent/futures/thread.py", line 172, in submit self._adjust_thread_count() File "/usr/lib/python3.7/concurrent/futures/thread.py", line 193, in _adjust_thread_count t.start() File "/usr/lib/python3.7/threading.py", line 852, in start _start_new_thread(self._bootstrap, ()) RuntimeError: can't start new thread ``` The process seems to run but it doesn't respond to any connection, not even `status` requests. We handle this exception and allow it to continue operation.
This commit is contained in:
parent
de9219e646
commit
89e6745274
@ -637,13 +637,17 @@ class PooledMixIn(socketserver.ThreadingMixIn):
|
||||
|
||||
def process_request(self, request, client_address):
|
||||
self.active_requests[request] = doublethink.utcnow()
|
||||
future = self.pool.submit(
|
||||
self.process_request_thread, request, client_address)
|
||||
future.add_done_callback(
|
||||
lambda f: self.active_requests.pop(request, None))
|
||||
if future.done():
|
||||
# avoid theoretical timing issue, in case process_request_thread
|
||||
# managed to finish before future.add_done_callback() ran
|
||||
try:
|
||||
future = self.pool.submit(
|
||||
self.process_request_thread, request, client_address)
|
||||
future.add_done_callback(
|
||||
lambda f: self.active_requests.pop(request, None))
|
||||
if future.done():
|
||||
# avoid theoretical timing issue, in case process_request_thread
|
||||
# managed to finish before future.add_done_callback() ran
|
||||
self.active_requests.pop(request, None)
|
||||
except RuntimeError as exc:
|
||||
self.logger.error("Error processing request %s", str(exc))
|
||||
self.active_requests.pop(request, None)
|
||||
|
||||
def get_request(self):
|
||||
|
Loading…
x
Reference in New Issue
Block a user