backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge pull request #11 from internetarchive/conn-backoff

Browse Source 
retry quickly on first connection failure then...
This commit is contained in:
jkafader 2018-10-29 14:05:08 -07:00 committed by GitHub
commit 8f772d9d29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doublethink/rethinker.py
View File

@ -1,7 +1,7 @@
''' '''
doublethink/rethinker.py - rethinkdb connection-manager doublethink/rethinker.py - rethinkdb connection-manager
Copyright (C) 2015-2017 Internet Archive Copyright (C) 2015-2018 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -117,6 +117,7 @@ class Rethinker(object):
# https://github.com/rethinkdb/rethinkdb-example-webpy-blog/blob/master/model.py # https://github.com/rethinkdb/rethinkdb-example-webpy-blog/blob/master/model.py
# "Best practices: Managing connections: a connection per request" # "Best practices: Managing connections: a connection per request"
def _random_server_connection(self): def _random_server_connection(self):
retry_wait = 0.01
while True: while True:
server = random.choice(self.servers) server = random.choice(self.servers)
try: try:
@ -126,10 +127,11 @@ class Rethinker(object):
except ValueError: except ValueError:
return r.connect(host=server) return r.connect(host=server)
except Exception as e: except Exception as e:
self.logger.error( self.logger.warn(
'will keep trying after failure connecting to ' 'will keep trying after failure connecting to '
'rethinkdb server at %s: %s', server, e) 'rethinkdb server at %s: %s', server, e)
time.sleep(0.5) time.sleep(retry_wait)
retry_wait = min(retry_wait * 2, 10.0)
def wrap(self, delegate): def wrap(self, delegate):
if isinstance(delegate, (types.FunctionType, types.MethodType)): if isinstance(delegate, (types.FunctionType, types.MethodType)):

4
tests/test_rethinker.py
View File

@ -1,7 +1,7 @@
''' '''
tests_rethinker.py - unit tests for doublethink connection manager tests_rethinker.py - unit tests for doublethink connection manager
Copyright (C) 2015-2017 Internet Archive Copyright (C) 2015-2018 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
@ -43,7 +43,7 @@ class RethinkerForTesting(doublethink.Rethinker):
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def rr(): def rr():
rr = RethinkerForTesting() rr = RethinkerForTesting(servers=['localhost','notexists'])
try: try:
rr.db_drop("doublethink_test_db").run() rr.db_drop("doublethink_test_db").run()
except r.errors.ReqlOpFailedError: except r.errors.ReqlOpFailedError: