backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 08:04:49 +01:00
Code Issues Projects Releases Activity

tests: fix fakeredis patch not running on test_handlers,

Browse Source 
use exc str instead of repr for error message for consistency
all tests pass on py2 and py3 again!
This commit is contained in:
Ilya Kreymer 2016-03-26 22:32:21 -04:00
parent 0399cc1046
commit 017e9802f8
6 changed files with 50 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webagg/handlers.py
View File

@ -106,7 +106,7 @@ class ResourceHandler(IndexHandler):
return out_headers, resp, errs
except WbException as e:
last_exc = e
errs[str(loader)] = repr(e)
errs[str(loader)] = str(e)
if last_exc:
errs['last_exc'] = last_exc

4
webagg/inputrequest.py
View File

@ -3,8 +3,8 @@ from pywb.utils.loaders import LimitReader
from pywb.utils.statusandheaders import StatusAndHeadersParser
from six.moves.urllib.parse import urlsplit, quote
from six import iteritems
from io import BytesIO, StringIO
from six import iteritems, StringIO
from io import BytesIO
#=============================================================================

4
webagg/responseloader.py
View File

@ -125,7 +125,7 @@ class RedisResolver(RedisIndexSource):
redis_key = cdx._formatter.format(redis_key)
res = self.redis.hget(redis_key, filename)
if res:
if res and six.PY3:
res = res.decode('utf-8')
return res
@ -135,7 +135,7 @@ class RedisResolver(RedisIndexSource):
class WARCPathLoader(BaseLoader):
def __init__(self, paths, cdx_source):
self.paths = paths
if isinstance(paths, str):
if isinstance(paths, six.string_types):
self.paths = [paths]
self.resolvers = [self._make_resolver(path) for path in self.paths]

2
webagg/test/test_dir_agg.py
View File

@ -207,7 +207,7 @@ class TestDirAgg(TempDirTests, BaseTestClass):
new_file = os.path.join(self.root_dir, 'colls/C/indexes/empty.cdxj')
with open(new_file, 'a') as fh:
os.utime(new_file)
os.utime(new_file, None)
res = self.cache_dir_loader.get_source_list({'url': 'example.com/', 'param.coll': '*'})

92
webagg/test/test_handlers.py
View File

@ -29,48 +29,40 @@ sources = {
'live': LiveIndexSource(),
}
testapp = None
def setup_module(self):
live_source = SimpleAggregator({'live': LiveIndexSource()})
live_handler = DefaultResourceHandler(live_source)
app = ResAggApp()
app.add_route('/live', live_handler)
source1 = GeventTimeoutAggregator(sources)
handler1 = DefaultResourceHandler(source1, to_path('testdata/'))
app.add_route('/many', handler1)
source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))})
handler2 = DefaultResourceHandler(source2, to_path('testdata/'))
app.add_route('/posttest', handler2)
source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))})
handler3 = DefaultResourceHandler(source3, to_path('testdata/'))
app.add_route('/fallback', HandlerSeq([handler3,
handler2,
live_handler]))
app.add_route('/seq', HandlerSeq([handler3,
handler2]))
app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
app.add_route('/empty', HandlerSeq([]))
app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
global testapp
testapp = webtest.TestApp(app.application)
def to_json_list(text):
return list([json.loads(cdx) for cdx in text.rstrip().split('\n')])
class TestResAgg(FakeRedisTests, BaseTestClass):
def setup(self):
self.testapp = testapp
def setup_class(cls):
super(TestResAgg, cls).setup_class()
live_source = SimpleAggregator({'live': LiveIndexSource()})
live_handler = DefaultResourceHandler(live_source)
app = ResAggApp()
app.add_route('/live', live_handler)
source1 = GeventTimeoutAggregator(sources)
handler1 = DefaultResourceHandler(source1, to_path('testdata/'))
app.add_route('/many', handler1)
source2 = SimpleAggregator({'post': FileIndexSource(to_path('testdata/post-test.cdxj'))})
handler2 = DefaultResourceHandler(source2, to_path('testdata/'))
app.add_route('/posttest', handler2)
source3 = SimpleAggregator({'example': FileIndexSource(to_path('testdata/example.cdxj'))})
handler3 = DefaultResourceHandler(source3, to_path('testdata/'))
app.add_route('/fallback', HandlerSeq([handler3,
handler2,
live_handler]))
app.add_route('/seq', HandlerSeq([handler3,
handler2]))
app.add_route('/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
app.add_route('/empty', HandlerSeq([]))
app.add_route('/invalid', DefaultResourceHandler([SimpleAggregator({'invalid': 'should not be a callable'})]))
cls.testapp = webtest.TestApp(app.application)
def _check_uri_date(self, resp, uri, dt):
buff = BytesIO(resp.body)
@ -128,10 +120,10 @@ class TestResAgg(FakeRedisTests, BaseTestClass):
resp = self.testapp.get('/live/index?url=http://httpbin.org/get&output=json')
resp.charset = 'utf-8'
res = to_json_list(resp.text)
res[0]['timestamp'] = '2016'
assert(res == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}])
cdxlist = list([json.loads(cdx) for cdx in resp.text.rstrip().split('\n')])
cdxlist[0]['timestamp'] = '2016'
assert(cdxlist == [{'url': 'http://httpbin.org/get', 'urlkey': 'org,httpbin)/get', 'is_live': 'true',
'load_url': 'http://httpbin.org/get', 'source': 'live', 'timestamp': '2016'}])
def test_live_resource(self):
headers = {'foo': 'bar'}
@ -343,26 +335,26 @@ foo=bar&test=abc"""
f.hset('test:warc', 'example.warc.gz', './testdata/example2.warc.gz')
resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
assert resp.json['message'] == "example.warc.gz:[Errno 2] No such file or directory: './testdata/example2.warc.gz'"
assert resp.json['message'] == "example.warc.gz: [Errno 2] No such file or directory: './testdata/example2.warc.gz'"
f.hdel('test:warc', 'example.warc.gz')
resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
assert resp.json == {'message': 'example.warc.gz: Archive File Not Found',
'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}}
f.delete('test:warc')
resp = self.testapp.get('/allredis/resource?url=http://www.example.com/', status=503)
assert resp.json == {'message': 'example.warc.gz:Archive File Not Found',
'errors': {'WARCPathLoader': "ArchiveLoadFailed('example.warc.gz:Archive File Not Found',)"}}
assert resp.json == {'message': 'example.warc.gz: Archive File Not Found',
'errors': {'WARCPathLoader': 'example.warc.gz: Archive File Not Found'}}
def test_error_fallback_live_not_found(self):
resp = self.testapp.get('/fallback/resource?url=http://invalid.url-not-found', status=400)
assert resp.json == {'message': 'http://invalid.url-not-found',
'errors': {'LiveWebLoader': "LiveResourceException('http://invalid.url-not-found',)"}}
'errors': {'LiveWebLoader': 'http://invalid.url-not-found'}}
assert resp.text == resp.headers['ResErrors']

8
webagg/test/test_inputreq.py
View File

@ -1,5 +1,5 @@
from webagg.inputrequest import DirectWSGIInputRequest, POSTInputRequest
from bottle import Bottle, request, response
from bottle import Bottle, request, response, debug
import webtest
import traceback
@ -8,7 +8,7 @@ import traceback
class InputReqApp(object):
def __init__(self):
self.application = Bottle()
self.application.default_error_handler = self.err_handler
debug(True)
@self.application.route('/test/<url:re:.*>', 'ANY')
def direct_input_request(url=''):
@ -23,10 +23,6 @@ class InputReqApp(object):
response['Content-Type'] = 'text/plain; charset=utf-8'
return inputreq.reconstruct_request(params.get('url'))
def err_handler(self, out):
print(out)
traceback.print_exc()
#=============================================================================
class TestInputReq(object):