webagg: add tests for RedisPathResolver and errors on missing warc, missing warc keys

webagg/responseloader.py

@@ -99,6 +99,7 @@ class RedisResolver(RedisIndexSource):
             redis_key = cdx._formatter.format(redis_key)
 
         res = self.redis.hget(redis_key, filename)
+        print('REDIS_KEY', redis_key, filename, res)
         if res:
             res = res.decode('utf-8')
 

webagg/test/test_handlers.py

@@ -17,6 +17,9 @@ from io import BytesIO
 
 import webtest
 
+from fakeredis import FakeStrictRedis
+from mock import patch
+
 from .testutils import to_path
 
 import json
@@ -30,6 +33,9 @@ sources = {
 
 testapp = None
 
+redismock = patch('redis.StrictRedis', FakeStrictRedis)
+redismock.start()
+
 def setup_module(self):
     live_source = SimpleAggregator({'live': LiveIndexSource()})
     live_handler = DefaultResourceHandler(live_source)
@@ -54,6 +60,8 @@ def setup_module(self):
     app.add_route('/seq', HandlerSeq([handler3,
                                   handler2]))
 
+    app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
+
     app.add_route('/empty', HandlerSeq([]))
     app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
 
@@ -61,6 +69,10 @@ def setup_module(self):
     testapp = webtest.TestApp(app.application)
 
 
+def teardown_module(self):
+    redismock.stop()
+
+
 def to_json_list(text):
     return list([json.loads(cdx) for cdx in text.rstrip().split('\n')])
 
@@ -88,6 +100,7 @@ class TestResAgg(object):
                                        '/many', '/many/postreq',
                                        '/posttest', '/posttest/postreq',
                                        '/seq', '/seq/postreq',
+                                       '/allredis', '/allredis/postreq',
                                        '/invalid', '/invalid/postreq'])
 
         assert res['/fallback'] == {'modes': ['list_sources', 'index', 'resource']}
@@ -326,6 +339,34 @@ foo=bar&test=abc"""
 
         assert 'ResErrors' not in resp.headers
 
+    def test_redis_warc_1(self):
+        f = FakeStrictRedis.from_url('redis://localhost/2')
+        f.hset('test:warc', 'example.warc.gz', './testdata/example.warc.gz')
+
+        resp = self.testapp.get('/allredis/resource?url=http://www.example.com/')
+
+        assert resp.headers['WebAgg-Source-Coll'] == 'example'
+
+    def test_error_redis_file_not_found(self):
+        f = FakeStrictRedis.from_url('redis://localhost/2')
+        f.hset('test:warc', 'example.warc.gz', './testdata/example2.warc.gz')
+
+        resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
+        assert resp.json['message'] == "example.warc.gz:[Errno 2] No such file or directory: './testdata/example2.warc.gz'"
+
+        f.hdel('test:warc', 'example.warc.gz')
+        resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
+
+        assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
+                             'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
+
+        f.delete('test:warc')
+        resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
+
+        assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
+                             'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
+
+
     def test_error_fallback_live_not_found(self):
         resp = self.testapp.get('/fallback/resource?url=http://invalid.url-not-found', status=400)