From 8b37fef8e0e09a5174a08f2ab6fd0f86450f554f Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 2 Apr 2014 15:01:40 -0700 Subject: [PATCH] tests: add explicit cdxserver config testing with different config variations --- pywb/cdx/cdxserver.py | 14 +++----------- pywb/cdx/cdxsource.py | 11 +++++++++-- pywb/cdx/test/test_cdxconfig.yaml | 2 ++ pywb/cdx/zipnum.py | 4 ++++ 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 pywb/cdx/test/test_cdxconfig.yaml diff --git a/pywb/cdx/cdxserver.py b/pywb/cdx/cdxserver.py index 77fe5e6e..137a2555 100644 --- a/pywb/cdx/cdxserver.py +++ b/pywb/cdx/cdxserver.py @@ -76,7 +76,7 @@ class BaseCDXServer(object): return self._check_cdx_iter(cdx_iter, query) - def _load_cdx_query(self, query): + def _load_cdx_query(self, query): # pragma: no cover raise NotImplementedError('Implement in subclass') @staticmethod @@ -148,9 +148,6 @@ class CDXServer(BaseCDXServer): self.sources.append(source) def add_cdx_source(self, source, config): - if source is None: - return - if isinstance(source, CDXSource): self._add_cdx_source(source) @@ -179,9 +176,6 @@ class CDXServer(BaseCDXServer): logging.warn('skipping unrecognized URI:%s', filename) return None - def __str__(self): - return 'CDX server serving from ' + str(self.sources) - #================================================================= class RemoteCDXServer(BaseCDXServer): @@ -204,9 +198,6 @@ class RemoteCDXServer(BaseCDXServer): def _load_cdx_query(self, query): return cdx_load([self.source], query, process=False) - def __str__(self): - return 'Remote CDX server serving from ' + str(self.sources[0]) - #================================================================= def create_cdx_server(config, ds_rules_file=None, server_cls=None): @@ -222,7 +213,8 @@ def create_cdx_server(config, ds_rules_file=None, server_cls=None): logging.debug('CDX Surt-Ordered? ' + str(surt_ordered)) if not server_cls: - if isinstance(paths, str) and is_http(paths): + if ((isinstance(paths, str) and is_http(paths)) or + isinstance(paths, RemoteCDXSource)): server_cls = RemoteCDXServer else: server_cls = CDXServer diff --git a/pywb/cdx/cdxsource.py b/pywb/cdx/cdxsource.py index 97670365..bf57209d 100644 --- a/pywb/cdx/cdxsource.py +++ b/pywb/cdx/cdxsource.py @@ -16,7 +16,7 @@ class CDXSource(object): """ Represents any cdx index source """ - def load_cdx(self, query): + def load_cdx(self, query): # pragma: no cover raise NotImplementedError('Implement in subclass') @@ -82,7 +82,10 @@ class RemoteCDXSource(CDXSource): return iter(response) def __str__(self): - return 'Remote CDX Server: ' + self.remote_url + if self.remote_processing: + return 'Remote CDX Server: ' + self.remote_url + else: + return 'Remote CDX Source: ' + self.remote_url #================================================================= @@ -91,6 +94,7 @@ class RedisCDXSource(CDXSource): def __init__(self, redis_url, config=None): import redis + self.redis_url = redis_url self.redis = redis.StrictRedis.from_url(redis_url) self.key_prefix = self.DEFAULT_KEY_PREFIX @@ -114,3 +118,6 @@ class RedisCDXSource(CDXSource): key += ' ' cdx_list = itertools.imap(lambda x: key + x, cdx_list) return cdx_list + + def __str__(self): + return 'Redis - ' + self.redis_url diff --git a/pywb/cdx/test/test_cdxconfig.yaml b/pywb/cdx/test/test_cdxconfig.yaml new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/pywb/cdx/test/test_cdxconfig.yaml @@ -0,0 +1,2 @@ + + diff --git a/pywb/cdx/zipnum.py b/pywb/cdx/zipnum.py index 66ced245..e282dfc0 100644 --- a/pywb/cdx/zipnum.py +++ b/pywb/cdx/zipnum.py @@ -200,3 +200,7 @@ class ZipNumCluster(CDXSource): end = query.end_key iter_ = itertools.takewhile(lambda line: line < end, iter_) return iter_ + + def __str__(self): + return 'ZipNum Cluster: {0}, {1}'.format(self.summary, + self.loc_filename)