diff --git a/webagg/handlers.py b/webagg/handlers.py index d9c06f96..83c22926 100644 --- a/webagg/handlers.py +++ b/webagg/handlers.py @@ -106,7 +106,7 @@ class ResourceHandler(IndexHandler): return out_headers, resp, errs except WbException as e: last_exc = e - errs[str(loader)] = repr(e) + errs[str(loader)] = str(e) if last_exc: errs['last_exc'] = last_exc diff --git a/webagg/inputrequest.py b/webagg/inputrequest.py index 19ff1960..39d12dd0 100644 --- a/webagg/inputrequest.py +++ b/webagg/inputrequest.py @@ -3,8 +3,8 @@ from pywb.utils.loaders import LimitReader from pywb.utils.statusandheaders import StatusAndHeadersParser from six.moves.urllib.parse import urlsplit, quote -from six import iteritems -from io import BytesIO, StringIO +from six import iteritems, StringIO +from io import BytesIO #============================================================================= diff --git a/webagg/responseloader.py b/webagg/responseloader.py index de6be389..9a619fa8 100644 --- a/webagg/responseloader.py +++ b/webagg/responseloader.py @@ -125,7 +125,7 @@ class RedisResolver(RedisIndexSource): redis_key = cdx._formatter.format(redis_key) res = self.redis.hget(redis_key, filename) - if res: + if res and six.PY3: res = res.decode('utf-8') return res @@ -135,7 +135,7 @@ class RedisResolver(RedisIndexSource): class WARCPathLoader(BaseLoader): def __init__(self, paths, cdx_source): self.paths = paths - if isinstance(paths, str): + if isinstance(paths, six.string_types): self.paths = [paths] self.resolvers = [self._make_resolver(path) for path in self.paths] diff --git a/webagg/test/test_dir_agg.py b/webagg/test/test_dir_agg.py index 0da78bf3..bce07046 100644 --- a/webagg/test/test_dir_agg.py +++ b/webagg/test/test_dir_agg.py @@ -207,7 +207,7 @@ class TestDirAgg(TempDirTests, BaseTestClass): new_file = os.path.join(self.root_dir, 'colls/C/indexes/empty.cdxj') with open(new_file, 'a') as fh: - os.utime(new_file) + os.utime(new_file, None) res = self.cache_dir_loader.get_source_list({'url': 'example.com/', 'param.coll': '*'}) diff --git a/webagg/test/test_handlers.py b/webagg/test/test_handlers.py index b28b31f9..1872e896 100644 --- a/webagg/test/test_handlers.py +++ b/webagg/test/test_handlers.py @@ -29,48 +29,40 @@ sources = { 'live': LiveIndexSource(), } -testapp = None - -def setup_module(self): - live_source = SimpleAggregator({'live': LiveIndexSource()}) - live_handler = DefaultResourceHandler(live_source) - app = ResAggApp() - app.add_route('/live', live_handler) - - source1 = GeventTimeoutAggregator(sources) - handler1 = DefaultResourceHandler(source1, to_path('testdata/')) - app.add_route('/many', handler1) - - source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))}) - handler2 = DefaultResourceHandler(source2, to_path('testdata/')) - app.add_route('/posttest', handler2) - - source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))}) - handler3 = DefaultResourceHandler(source3, to_path('testdata/')) - - app.add_route('/fallback', HandlerSeq([handler3, - handler2, - live_handler])) - - app.add_route('/seq', HandlerSeq([handler3, - handler2])) - - app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc')) - - app.add_route('/empty', HandlerSeq([])) - app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})])) - - global testapp - testapp = webtest.TestApp(app.application) - - -def to_json_list(text): - return list([json.loads(cdx) for cdx in text.rstrip().split('\n')]) - class TestResAgg(FakeRedisTests, BaseTestClass): - def setup(self): - self.testapp = testapp + def setup_class(cls): + super(TestResAgg, cls).setup_class() + + live_source = SimpleAggregator({'live': LiveIndexSource()}) + live_handler = DefaultResourceHandler(live_source) + app = ResAggApp() + app.add_route('/live', live_handler) + + source1 = GeventTimeoutAggregator(sources) + handler1 = DefaultResourceHandler(source1, to_path('testdata/')) + app.add_route('/many', handler1) + + source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))}) + handler2 = DefaultResourceHandler(source2, to_path('testdata/')) + app.add_route('/posttest', handler2) + + source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))}) + handler3 = DefaultResourceHandler(source3, to_path('testdata/')) + + app.add_route('/fallback', HandlerSeq([handler3, + handler2, + live_handler])) + + app.add_route('/seq', HandlerSeq([handler3, + handler2])) + + app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc')) + + app.add_route('/empty', HandlerSeq([])) + app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})])) + + cls.testapp = webtest.TestApp(app.application) def _check_uri_date(self, resp, uri, dt): buff = BytesIO(resp.body) @@ -128,10 +120,10 @@ class TestResAgg(FakeRedisTests, BaseTestClass): resp = self.testapp.get('/live/index?url=http://httpbin.org/get&output=json') resp.charset = 'utf-8' - res = to_json_list(resp.text) - res[0]['timestamp'] = '2016' - assert(res == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true', - 'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}]) + cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')]) + cdxlist[0]['timestamp'] = '2016' + assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true', + 'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}]) def test_live_resource(self): headers = {'foo': 'bar'} @@ -343,26 +335,26 @@ foo=bar&test=abc""" f.hset('test:warc', 'example.warc.gz', './testdata/example2.warc.gz') resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) - assert resp.json['message'] == "example.warc.gz:[Errno 2] No such file or directory: './testdata/example2.warc.gz'" + assert resp.json['message'] == "example.warc.gz: [Errno 2] No such file or directory: './testdata/example2.warc.gz'" f.hdel('test:warc', 'example.warc.gz') resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) - assert resp.json == {'message': 'example.warc.gz:Archive File Not Found', - 'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}} + assert resp.json == {'message': 'example.warc.gz: Archive File Not Found', + 'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}} f.delete('test:warc') resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503) - assert resp.json == {'message': 'example.warc.gz:Archive File Not Found', - 'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}} + assert resp.json == {'message': 'example.warc.gz: Archive File Not Found', + 'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}} def test_error_fallback_live_not_found(self): resp = self.testapp.get('/fallback/resource?url=http://invalid.url-not-found', status=400) assert resp.json == {'message': 'http://invalid.url-not-found', - 'errors': {'LiveWebLoader': "LiveResourceException('http://invalid.url-not-found',)"}} + 'errors': {'LiveWebLoader': 'http://invalid.url-not-found'}} assert resp.text == resp.headers['ResErrors'] diff --git a/webagg/test/test_inputreq.py b/webagg/test/test_inputreq.py index 7aca5b6a..bdc47705 100644 --- a/webagg/test/test_inputreq.py +++ b/webagg/test/test_inputreq.py @@ -1,5 +1,5 @@ from webagg.inputrequest import DirectWSGIInputRequest, POSTInputRequest -from bottle import Bottle, request, response +from bottle import Bottle, request, response, debug import webtest import traceback @@ -8,7 +8,7 @@ import traceback class InputReqApp(object): def __init__(self): self.application = Bottle() - self.application.default_error_handler = self.err_handler + debug(True) @self.application.route('/test/', 'ANY') def direct_input_request(url=''): @@ -23,10 +23,6 @@ class InputReqApp(object): response['Content-Type'] = 'text/plain; charset=utf-8' return inputreq.reconstruct_request(params.get('url')) - def err_handler(self, out): - print(out) - traceback.print_exc() - #============================================================================= class TestInputReq(object):