From 8ec2d853b6fde8fab50fb8e59df912479dcf7eca Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 1 Nov 2018 19:17:50 +0000 Subject: [PATCH] connect quickly when a server is down Another tweak to that end. We have observed that when a rethinkdb server is offline, an attempt to connect to it takes a second or two to time out. On the other hand, if the host is up but the port is not open (rethinkdb is not running or something like that), the connection failure happens very quickly. To achieve good performance in case a rethinkdb server is down, we are now setting a timeout on the connect() call. The timeout starts at 0.1 sec, for quick retry, and backs off up to 10 sec in case of repeated failures. --- doublethink/rethinker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doublethink/rethinker.py b/doublethink/rethinker.py index 1538c0c..fcafa20 100644 --- a/doublethink/rethinker.py +++ b/doublethink/rethinker.py @@ -123,13 +123,15 @@ class Rethinker(object): try: try: host, port = server.split(':') - return r.connect(host=host, port=port) + return r.connect( + host=host, port=port, timeout=max(0.1, retry_wait)) except ValueError: - return r.connect(host=server) + return r.connect(host=server, timeout=max(0.1, retry_wait)) except Exception as e: self.logger.warn( 'will keep trying after failure connecting to ' - 'rethinkdb server at %s: %s', server, e) + 'rethinkdb server at %s: %s (sleeping for %s sec)', + server, e, retry_wait) time.sleep(retry_wait) retry_wait = min(retry_wait * 2, 10.0)