diff --git a/webagg/responseloader.py b/webagg/responseloader.py index 33228444..e600533e 100644 --- a/webagg/responseloader.py +++ b/webagg/responseloader.py @@ -99,6 +99,7 @@ class RedisResolver(RedisIndexSource): redis_key = cdx._formatter.format(redis_key) res = self.redis.hget(redis_key, filename) + print('REDIS_KEY', redis_key, filename, res) if res: res = res.decode('utf-8') diff --git a/webagg/test/test_handlers.py b/webagg/test/test_handlers.py index cefaa99d..9ffa26b2 100644 --- a/webagg/test/test_handlers.py +++ b/webagg/test/test_handlers.py @@ -17,6 +17,9 @@ from io import BytesIO import webtest +from fakeredis import FakeStrictRedis +from mock import patch + from .testutils import to_path import json @@ -30,6 +33,9 @@ sources = { testapp = None +redismock = patch('redis.StrictRedis', FakeStrictRedis) +redismock.start() + def setup_module(self): live_source = SimpleAggregator({'live': LiveIndexSource()}) live_handler = DefaultResourceHandler(live_source) @@ -54,6 +60,8 @@ def setup_module(self): app.add_route('/seq', HandlerSeq([handler3, handler2])) + app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc')) + app.add_route('/empty', HandlerSeq([])) app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})])) @@ -61,6 +69,10 @@ def setup_module(self): testapp = webtest.TestApp(app.application) +def teardown_module(self): + redismock.stop() + + def to_json_list(text): return list([json.loads(cdx) for cdx in text.rstrip().split('\n')]) @@ -88,6 +100,7 @@ class TestResAgg(object): '/many', '/many/postreq', '/posttest', '/posttest/postreq', '/seq', '/seq/postreq', + '/allredis', '/allredis/postreq', '/invalid', '/invalid/postreq']) assert res['/fallback'] == {'modes': ['list_sources', 'index', 'resource']} @@ -326,6 +339,34 @@ foo=bar&test=abc""" assert 'ResErrors' not in resp.headers + def test_redis_warc_1(self): + f = FakeStrictRedis.from_url('redis://localhost/2') + f.hset('test:warc', 'example.warc.gz', './testdata/example.warc.gz') + + resp = self.testapp.get('/allredis/resource?url=http://www.example.com/') + + assert resp.headers['WebAgg-Source-Coll'] == 'example' + + def test_error_redis_file_not_found(self): + f = FakeStrictRedis.from_url('redis://localhost/2') + f.hset('test:warc', 'example.warc.gz', './testdata/example2.warc.gz') + + resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) + assert resp.json['message'] == "example.warc.gz:[Errno 2] No such file or directory: './testdata/example2.warc.gz'" + + f.hdel('test:warc', 'example.warc.gz') + resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) + + assert resp.json == {'message': 'example.warc.gz:Archive File Not Found', + 'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}} + + f.delete('test:warc') + resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) + + assert resp.json == {'message': 'example.warc.gz:Archive File Not Found', + 'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}} + + def test_error_fallback_live_not_found(self): resp = self.testapp.get('/fallback/resource?url=http://invalid.url-not-found', status=400)