backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge pull request #11 from internetarchive/conn-backoff

Browse Source 
retry quickly on first connection failure then...
This commit is contained in:
jkafader 2018-10-29 14:05:08 -07:00 committed by GitHub
commit 8f772d9d29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doublethink/rethinker.py
View File

@ -1,7 +1,7 @@
'''
doublethink/rethinker.py - rethinkdb connection-manager
Copyright (C) 2015-2017 Internet Archive
Copyright (C) 2015-2018 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -117,6 +117,7 @@ class Rethinker(object):
# https://github.com/rethinkdb/rethinkdb-example-webpy-blog/blob/master/model.py
# "Best practices: Managing connections: a connection per request"
def _random_server_connection(self):
retry_wait = 0.01
while True:
server = random.choice(self.servers)
try:
@ -126,10 +127,11 @@ class Rethinker(object):
except ValueError:
return r.connect(host=server)
except Exception as e:
self.logger.error(
self.logger.warn(
'will keep trying after failure connecting to '
'rethinkdb server at %s: %s', server, e)
time.sleep(0.5)
time.sleep(retry_wait)
retry_wait = min(retry_wait * 2, 10.0)
def wrap(self, delegate):
if isinstance(delegate, (types.FunctionType, types.MethodType)):

4
tests/test_rethinker.py
View File

@ -1,7 +1,7 @@
'''
tests_rethinker.py - unit tests for doublethink connection manager
Copyright (C) 2015-2017 Internet Archive
Copyright (C) 2015-2018 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -43,7 +43,7 @@ class RethinkerForTesting(doublethink.Rethinker):
@pytest.fixture(scope="module")
def rr():
rr = RethinkerForTesting()
rr = RethinkerForTesting(servers=['localhost','notexists'])
try:
rr.db_drop("doublethink_test_db").run()
except r.errors.ReqlOpFailedError: