backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

config limit: add query_limit config options to specify optional limit for both exact and prefix queries, addresses ukwa/ukwa-pywb#49 (#518)

Browse Source
This commit is contained in:
Ilya Kreymer 2019-11-07 10:25:49 -08:00 committed by GitHub
parent fe09d9991e
commit 66ac3ca114
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pywb/apps/frontendapp.py
View File

@ -92,6 +92,7 @@ class FrontEndApp(object):
self.static_handler = StaticHandler(static_path) self.static_handler = StaticHandler(static_path)
self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx') self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx')
self.query_limit = config.get('query_limit')
upstream_paths = self.get_upstream_paths(self.warcserver_server.port) upstream_paths = self.get_upstream_paths(self.warcserver_server.port)
@ -355,6 +356,10 @@ class FrontEndApp(object):
cdx_url += '&' if '?' in cdx_url else '?' cdx_url += '&' if '?' in cdx_url else '?'
cdx_url += environ.get('QUERY_STRING') cdx_url += environ.get('QUERY_STRING')
if self.query_limit:
cdx_url += '&' if '?' in cdx_url else '?'
cdx_url += 'limit=' + str(self.query_limit)
try: try:
res = requests.get(cdx_url, stream=True) res = requests.get(cdx_url, stream=True)

3
pywb/apps/rewriterapp.py
View File

@ -713,6 +713,9 @@ class RewriterApp(object):
if 'memento_format' in kwargs: if 'memento_format' in kwargs:
params['memento_format'] = kwargs['memento_format'] params['memento_format'] = kwargs['memento_format']
if 'limit' in kwargs:
params['limit'] = kwargs['limit']
upstream_url = self.get_upstream_url(wb_url, kwargs, params) upstream_url = self.get_upstream_url(wb_url, kwargs, params)
upstream_url = upstream_url.replace('/resource/postreq', '/index') upstream_url = upstream_url.replace('/resource/postreq', '/index')

2
tests/config_test_redirect_classic.yaml
View File

@ -17,3 +17,5 @@ enable_memento: true
enable_prefer: true enable_prefer: true
debug: true debug: true
query_limit: 10

4
tests/test_redirect_classic.py
View File

@ -74,4 +74,8 @@ class TestRedirectClassic(BaseConfigTest):
resp = self.get('/live/{0}http://example.com/?test=test', fmod_slash) resp = self.get('/live/{0}http://example.com/?test=test', fmod_slash)
assert resp.status_int == 200 assert resp.status_int == 200
def test_replay_limit_cdx(self):
resp = self.testapp.get('/pywb/cdx?url=http://www.iana.org/*&output=json')
assert resp.content_type == 'text/x-ndjson'
assert len(resp.text.rstrip().split('\n')) == 10