diff --git a/docs/manual/warcserver.rst b/docs/manual/warcserver.rst index ce99e125..4c47eba7 100644 --- a/docs/manual/warcserver.rst +++ b/docs/manual/warcserver.rst @@ -216,8 +216,13 @@ lookup three remote archives, two using memento, and one using CDX Server API:: ia: cdx+http://web.archive.org/cdx;/web apt: memento+http://arquivo.pt/wayback/ + timeout: 10 + This allows Warcserver to serve as a "Memento Aggregator", aggregating results from -multiple existing archives (using the Memento API and other APIs) +multiple existing archives (using the Memento API and other APIs). + +An optional ``timeout`` property configures how many seconds to wait for each source before +it is considered to have 'timed out'. (If unspecified, the default value is 5 seconds). Sequential Fallback Collections """"""""""""""""""""""""""""""" diff --git a/pywb/warcserver/index/aggregator.py b/pywb/warcserver/index/aggregator.py index fd3c806b..16cd25e9 100644 --- a/pywb/warcserver/index/aggregator.py +++ b/pywb/warcserver/index/aggregator.py @@ -220,10 +220,12 @@ class TimeoutMixin(object): #============================================================================= class GeventMixin(object): + DEFAULT_TIMEOUT = 5.0 + def __init__(self, *args, **kwargs): super(GeventMixin, self).__init__(*args, **kwargs) self.pool = Pool(size=kwargs.get('size')) - self.timeout = kwargs.get('timeout', 5.0) + self.timeout = kwargs.get('timeout') or self.DEFAULT_TIMEOUT def _load_all(self, params): params['_timeout'] = self.timeout diff --git a/pywb/warcserver/index/test/test_memento_agg.py b/pywb/warcserver/index/test/test_memento_agg.py index 20cf4838..b621fea6 100644 --- a/pywb/warcserver/index/test/test_memento_agg.py +++ b/pywb/warcserver/index/test/test_memento_agg.py @@ -34,7 +34,7 @@ aggs_inv = {'simple': SimpleAggregator(sources, invert_sources=True), 'gevent': GeventTimeoutAggregator(sources, invert_sources=True, timeout=5.0), } -agg_tm = {'gevent': GeventTimeoutAggregator(sources, timeout=0.0)} +agg_tm = {'gevent': GeventTimeoutAggregator(sources, timeout=0.05)} nf = {'notfound': FileIndexSource('testdata/not-found-x')} agg_nf = {'simple': SimpleAggregator(nf), diff --git a/pywb/warcserver/test/test_warcserver_config.yaml b/pywb/warcserver/test/test_warcserver_config.yaml index cc61b58f..c6b0dfb6 100644 --- a/pywb/warcserver/test/test_warcserver_config.yaml +++ b/pywb/warcserver/test/test_warcserver_config.yaml @@ -43,6 +43,8 @@ collections: apt: memento+http://arquivo.pt/wayback/ liveweb: live + timeout: 10 + # Local Dir CDX local: index: ./local/indexes