aggregator timeout fixes (#310):

- fix memento aggregation if timeout is 0.0
- use default timeout (5.0), instead of default to 0.0 and failing
- add 'timeout' property to warcserver aggregation tests
- docs: mention property in warcserver docs also

docs/manual/warcserver.rst

@@ -216,8 +216,13 @@ lookup three remote archives, two using memento, and one using CDX Server API::
         ia:   cdx+http://web.archive.org/cdx;/web
         apt:  memento+http://arquivo.pt/wayback/
 
+      timeout: 10
+
 This allows Warcserver to serve as a "Memento Aggregator", aggregating results from
-multiple existing archives (using the Memento API and other APIs)
+multiple existing archives (using the Memento API and other APIs).
+
+An optional ``timeout`` property configures how many seconds to wait for each source before
+it is considered to have 'timed out'. (If unspecified, the default value is 5 seconds).
 
 Sequential Fallback Collections
 """""""""""""""""""""""""""""""

pywb/warcserver/index/aggregator.py

@@ -220,10 +220,12 @@ class TimeoutMixin(object):
 
 #=============================================================================
 class GeventMixin(object):
+    DEFAULT_TIMEOUT = 5.0
+
     def __init__(self, *args, **kwargs):
         super(GeventMixin, self).__init__(*args, **kwargs)
         self.pool = Pool(size=kwargs.get('size'))
-        self.timeout = kwargs.get('timeout', 5.0)
+        self.timeout = kwargs.get('timeout') or self.DEFAULT_TIMEOUT
 
     def _load_all(self, params):
         params['_timeout'] = self.timeout

pywb/warcserver/index/test/test_memento_agg.py

@@ -34,7 +34,7 @@ aggs_inv = {'simple': SimpleAggregator(sources, invert_sources=True),
             'gevent': GeventTimeoutAggregator(sources, invert_sources=True, timeout=5.0),
            }
 
-agg_tm = {'gevent': GeventTimeoutAggregator(sources, timeout=0.0)}
+agg_tm = {'gevent': GeventTimeoutAggregator(sources, timeout=0.05)}
 
 nf = {'notfound': FileIndexSource('testdata/not-found-x')}
 agg_nf = {'simple': SimpleAggregator(nf),

pywb/warcserver/test/test_warcserver_config.yaml

@@ -43,6 +43,8 @@ collections:
             apt:  memento+http://arquivo.pt/wayback/
             liveweb: live
 
+        timeout: 10
+
     # Local Dir CDX
     local:
         index: ./local/indexes