tests: fix fakeredis patch not running on test_handlers,

use exc str instead of repr for error message for consistency
all tests pass on py2 and py3 again!

webagg/handlers.py

@@ -106,7 +106,7 @@ class ResourceHandler(IndexHandler):
                         return out_headers, resp, errs
                 except WbException as e:
                     last_exc = e
-                    errs[str(loader)] = repr(e)
+                    errs[str(loader)] = str(e)
 
         if last_exc:
             errs['last_exc'] = last_exc

webagg/inputrequest.py

@@ -3,8 +3,8 @@ from pywb.utils.loaders import LimitReader
 from pywb.utils.statusandheaders import StatusAndHeadersParser
 
 from six.moves.urllib.parse import urlsplit, quote
-from six import iteritems
+from six import iteritems, StringIO
-from io import BytesIO, StringIO
+from io import BytesIO
 
 
 #=============================================================================

webagg/responseloader.py

@@ -125,7 +125,7 @@ class RedisResolver(RedisIndexSource):
             redis_key = cdx._formatter.format(redis_key)
 
         res = self.redis.hget(redis_key, filename)
-        if res:
+        if res and six.PY3:
             res = res.decode('utf-8')
 
         return res
@@ -135,7 +135,7 @@ class RedisResolver(RedisIndexSource):
 class WARCPathLoader(BaseLoader):
     def __init__(self, paths, cdx_source):
         self.paths = paths
-        if isinstance(paths, str):
+        if isinstance(paths, six.string_types):
             self.paths = [paths]
 
         self.resolvers = [self._make_resolver(path) for path in self.paths]

webagg/test/test_dir_agg.py

@@ -207,7 +207,7 @@ class TestDirAgg(TempDirTests, BaseTestClass):
         new_file = os.path.join(self.root_dir, 'colls/C/indexes/empty.cdxj')
 
         with open(new_file, 'a') as fh:
-            os.utime(new_file)
+            os.utime(new_file, None)
 
         res = self.cache_dir_loader.get_source_list({'url': 'example.com/', 'param.coll': '*'})
 

webagg/test/test_handlers.py

@@ -29,48 +29,40 @@ sources = {
     'live': LiveIndexSource(),
 }
 
-testapp = None
-
-def setup_module(self):
-    live_source = SimpleAggregator({'live': LiveIndexSource()})
-    live_handler = DefaultResourceHandler(live_source)
-    app = ResAggApp()
-    app.add_route('/live', live_handler)
-
-    source1 = GeventTimeoutAggregator(sources)
-    handler1 = DefaultResourceHandler(source1, to_path('testdata/'))
-    app.add_route('/many', handler1)
-
-    source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))})
-    handler2 = DefaultResourceHandler(source2, to_path('testdata/'))
-    app.add_route('/posttest', handler2)
-
-    source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))})
-    handler3 = DefaultResourceHandler(source3, to_path('testdata/'))
-
-    app.add_route('/fallback', HandlerSeq([handler3,
-                                       handler2,
-                                       live_handler]))
-
-    app.add_route('/seq', HandlerSeq([handler3,
-                                  handler2]))
-
-    app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
-
-    app.add_route('/empty', HandlerSeq([]))
-    app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
-
-    global testapp
-    testapp = webtest.TestApp(app.application)
-
-
-def to_json_list(text):
-    return list([json.loads(cdx) for cdx in text.rstrip().split('\n')])
-
 
 class TestResAgg(FakeRedisTests, BaseTestClass):
-    def setup(self):
+    def setup_class(cls):
-        self.testapp = testapp
+        super(TestResAgg, cls).setup_class()
+
+        live_source = SimpleAggregator({'live': LiveIndexSource()})
+        live_handler = DefaultResourceHandler(live_source)
+        app = ResAggApp()
+        app.add_route('/live', live_handler)
+
+        source1 = GeventTimeoutAggregator(sources)
+        handler1 = DefaultResourceHandler(source1, to_path('testdata/'))
+        app.add_route('/many', handler1)
+
+        source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))})
+        handler2 = DefaultResourceHandler(source2, to_path('testdata/'))
+        app.add_route('/posttest', handler2)
+
+        source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))})
+        handler3 = DefaultResourceHandler(source3, to_path('testdata/'))
+
+        app.add_route('/fallback', HandlerSeq([handler3,
+                                           handler2,
+                                           live_handler]))
+
+        app.add_route('/seq', HandlerSeq([handler3,
+                                      handler2]))
+
+        app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
+
+        app.add_route('/empty', HandlerSeq([]))
+        app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
+
+        cls.testapp = webtest.TestApp(app.application)
 
     def _check_uri_date(self, resp, uri, dt):
         buff = BytesIO(resp.body)
@@ -128,10 +120,10 @@ class TestResAgg(FakeRedisTests, BaseTestClass):
         resp = self.testapp.get('/live/index?url=http://httpbin.org/get&output=json')
         resp.charset = 'utf-8'
 
-        res = to_json_list(resp.text)
+        cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')])
-        res[0]['timestamp'] = '2016'
+        cdxlist[0]['timestamp'] = '2016'
-        assert(res == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
+        assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
-                        'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}])
+                            'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}])
 
     def test_live_resource(self):
         headers = {'foo': 'bar'}
@@ -343,26 +335,26 @@ foo=bar&test=abc"""
         f.hset('test:warc', 'example.warc.gz', './testdata/example2.warc.gz')
 
         resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
-        assert resp.json['message'] == "example.warc.gz:[Errno 2] No such file or directory: './testdata/example2.warc.gz'"
+        assert resp.json['message'] == "example.warc.gz: [Errno 2] No such file or directory: './testdata/example2.warc.gz'"
 
         f.hdel('test:warc', 'example.warc.gz')
         resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
 
-        assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
+        assert resp.json == {'message': 'example.warc.gz: Archive File Not Found',
-                             'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
+                             'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}}
 
         f.delete('test:warc')
         resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
 
-        assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
+        assert resp.json == {'message': 'example.warc.gz: Archive File Not Found',
-                             'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
+                             'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}}
 
 
     def test_error_fallback_live_not_found(self):
         resp = self.testapp.get('/fallback/resource?url=http://invalid.url-not-found', status=400)
 
         assert resp.json == {'message': 'http://invalid.url-not-found',
-                             'errors': {'LiveWebLoader': "LiveResourceException('http://invalid.url-not-found',)"}}
+                             'errors': {'LiveWebLoader': 'http://invalid.url-not-found'}}
 
         assert resp.text == resp.headers['ResErrors']
 

webagg/test/test_inputreq.py

@@ -1,5 +1,5 @@
 from webagg.inputrequest import DirectWSGIInputRequest, POSTInputRequest
-from bottle import Bottle, request, response
+from bottle import Bottle, request, response, debug
 import webtest
 import traceback
 
@@ -8,7 +8,7 @@ import traceback
 class InputReqApp(object):
     def __init__(self):
         self.application = Bottle()
-        self.application.default_error_handler = self.err_handler
+        debug(True)
 
         @self.application.route('/test/<url:re:.*>', 'ANY')
         def direct_input_request(url=''):
@@ -23,10 +23,6 @@ class InputReqApp(object):
             response['Content-Type'] = 'text/plain; charset=utf-8'
             return inputreq.reconstruct_request(params.get('url'))
 
-    def err_handler(self, out):
-        print(out)
-        traceback.print_exc()
-
 
 #=============================================================================
 class TestInputReq(object):