backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-14 15:53:28 +01:00
Code Issues Projects Releases Activity

Upgrade dependencies (#839)

Browse Source 
- Update and pin dependencies to specific versions that support Python 3.7-3.11
- Replace deprecated werkzeug.pop_path_info with wsgiref.shift_path_info
- Use the latest httpbin from psf/httpbin
- Remove unused flask test dependency
- Drop Python 2 and Python <3.7 support
- Ensure greenlet 2 is used for now, as psf/httpbin doesn't yet work with greenlet 3

---------

Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>
This commit is contained in:
Ed Summers 2024-04-02 17:16:50 -04:00 committed by GitHub
parent f40e7ef18c
commit b4955cca66
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 46 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yaml vendored
View File

@ -8,7 +8,7 @@ jobs:
strategy:
max-parallel: 3
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10']
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
steps:
- name: checkout

8
pywb/apps/frontendapp.py
View File

@ -1,7 +1,7 @@
from gevent.monkey import patch_all; patch_all()
from werkzeug.routing import Map, Rule, RequestRedirect, Submount
from werkzeug.wsgi import pop_path_info
from wsgiref.util import shift_path_info
from six.moves.urllib.parse import urljoin, parse_qsl
from six import iteritems
from warcio.utils import to_native_str
@ -558,9 +558,9 @@ class FrontEndApp(object):
return
if coll != '$root':
pop_path_info(environ)
shift_path_info(environ)
if record:
pop_path_info(environ)
shift_path_info(environ)
paths = [self.warcserver.root_dir]
@ -669,7 +669,7 @@ class FrontEndApp(object):
lang = args.pop('lang', '')
if lang:
pop_path_info(environ)
shift_path_info(environ)
if lang:
environ['pywb_lang'] = lang

2
pywb/apps/rewriterapp.py
View File

@ -64,7 +64,7 @@ class RewriterApp(object):
if not jinja_env:
jinja_env = JinjaEnv(globals={'static_path': 'static'},
extensions=['jinja2.ext.i18n', 'jinja2.ext.with_'])
extensions=['jinja2.ext.i18n'])
jinja_env.jinja_env.install_null_translations()
self.jinja_env = jinja_env

10
pywb/rewrite/templateview.py
View File

@ -5,7 +5,7 @@ from pywb.utils.loaders import load
from six.moves.urllib.parse import urlsplit, quote
from jinja2 import Environment, TemplateNotFound, contextfunction, select_autoescape
from jinja2 import Environment, TemplateNotFound, pass_context, select_autoescape
from jinja2 import FileSystemLoader, PackageLoader, ChoiceLoader
from webassets.ext.jinja2 import AssetsExtension
@ -139,7 +139,7 @@ class JinjaEnv(object):
return loc_map.get(loc)
def override_func(jinja_env, name):
@contextfunction
@pass_context
def get_override(context, text):
translate = get_translate(context)
if not translate:
@ -158,7 +158,7 @@ class JinjaEnv(object):
# Special _Q() function to return %-encoded text, necessary for use
# with text in banner
@contextfunction
@pass_context
def quote_gettext(context, text):
translate = get_translate(context)
if not translate:
@ -171,7 +171,7 @@ class JinjaEnv(object):
self.jinja_env.globals['_Q'] = quote_gettext
self.jinja_env.globals['default_locale'] = default_locale
@contextfunction
@pass_context
def switch_locale(context, locale):
environ = context.get('env')
curr_loc = environ.get('pywb_lang', '')
@ -188,7 +188,7 @@ class JinjaEnv(object):
return app_prefix + '/' + locale + request_uri
@contextfunction
@pass_context
def get_locale_prefixes(context):
environ = context.get('env')
locale_prefixes = {}

2
pywb/warcserver/test/test_inputreq.py
View File

@ -39,7 +39,7 @@ class InputReqApp(object):
#=============================================================================
class TestInputReq(object):
def setup(self):
def setup_method(self):
self.app = InputReqApp()
self.testapp = webtest.TestApp(self.app)

2
pywb/warcserver/test/test_upstream.py
View File

@ -18,7 +18,7 @@ from .testutils import LiveServerTests, HttpBinLiveTests, BaseTestClass
class TestUpstream(LiveServerTests, HttpBinLiveTests, BaseTestClass):
def setup(self):
def setup_method(self):
app = BaseWarcServer()
base_url = 'http://localhost:{0}'.format(self.server.port)

11
requirements.txt
View File

@ -1,19 +1,20 @@
six
warcio>=1.7.1
requests
redis<3.0
jinja2<3.0.0
redis
jinja2>=3.1.2
surt>=0.3.1
brotlipy
pyyaml
werkzeug
werkzeug==2.2.3
webencodings
gevent==21.12.0
gevent==22.10.2
greenlet>=2.0.2,<3.0
webassets==2.0
portalocker
wsgiprox>=1.5.1
fakeredis<1.0
tldextract
python-dateutil
markupsafe<2.1.0
markupsafe>=2.1.1
ua_parser

9
setup.py
View File

@ -113,6 +113,7 @@ setup(
"translate_toolkit"
],
},
python_requires='>=3.7,<3.12',
tests_require=load_requirements("test_requirements.txt"),
cmdclass={'test': PyTest},
test_suite='',
@ -131,16 +132,12 @@ setup(
'Environment :: Web Environment',
'License :: OSI Approved :: GNU General Public License (GPL)',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Topic :: Internet :: Proxy Servers',
'Topic :: Internet :: WWW/HTTP',
'Topic :: Internet :: WWW/HTTP :: WSGI',

3
test_requirements.txt
View File

@ -3,7 +3,6 @@ WebTest
pytest-cov
mock
urllib3
httpbin==0.5.0
flask<2.0
ujson
lxml
httpbin>=0.10.2

2
tests/test_force_https.py
View File

@ -56,6 +56,6 @@ class TestForceHttpsRoot(BaseConfigTest):
resp = self.get('/20140128051539{0}/http://www.iana.org/domains/example', fmod,
headers={'X-Forwarded-Proto': 'https'})
assert resp.headers['Location'] == 'https://localhost:80/20140128051539{0}/http://www.iana.org/domains/reserved'.format(fmod)
assert resp.headers['Location'] == 'https://localhost:80/20140128051539{0}/http://www.iana.org/help/example-domains'.format(fmod)

35
tests/test_live_rewriter.py
View File

@ -91,25 +91,28 @@ class TestLiveRewriter(HttpBinLiveTests, BaseConfigTest):
resp = self.head('/live/{0}httpbin.org/get?foo=bar', fmod_sl)
assert resp.status_int == 200
@pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
def test_live_bad_content_length(self, fmod_sl):
resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl, status=200)
assert resp.headers['Content-Length'] == '149'
# Following tests are temporarily commented out because latest version of PSF httpbin
# now returns 400 if content-length header isn't parsable as an int
resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl, status=200)
assert resp.headers['Content-Length'] == '90'
# @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
# def test_live_bad_content_length(self, fmod_sl):
# resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl, status=200)
# assert resp.headers['Content-Length'] == '149'
@pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
def test_live_bad_content_length_with_range(self, fmod_sl):
resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl,
headers={'Range': 'bytes=0-'}, status=206)
assert resp.headers['Content-Length'] == '149'
assert resp.headers['Content-Range'] == 'bytes 0-148/149'
# resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl, status=200)
# assert resp.headers['Content-Length'] == '90'
resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl,
headers={'Range': 'bytes=0-'}, status=206)
assert resp.headers['Content-Length'] == '90'
assert resp.headers['Content-Range'] == 'bytes 0-89/90'
# @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
# def test_live_bad_content_length_with_range(self, fmod_sl):
# resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl,
# headers={'Range': 'bytes=0-'}, status=206)
# assert resp.headers['Content-Length'] == '149'
# assert resp.headers['Content-Range'] == 'bytes 0-148/149'
# resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl,
# headers={'Range': 'bytes=0-'}, status=206)
# assert resp.headers['Content-Length'] == '90'
# assert resp.headers['Content-Range'] == 'bytes 0-89/90'
def test_custom_unicode_header(self, fmod_sl):
value = u'⛄'

6
tox.ini
View File

@ -4,15 +4,15 @@ testpaths =
tests
[tox]
envlist = py36, py37, py38, py39, py310
envlist = py37, py38, py39, py310, py311
[gh-actions]
python =
3.6: py36
3.7: py37
3.8: py38
3.9: py39
3.10: py310
3.11: py311
[testenv]
setenv = PYWB_NO_VERIFY_SSL = 1
@ -22,6 +22,6 @@ deps =
-rrequirements.txt
-rextra_requirements.txt
commands =
py.test --cov-config .coveragerc --cov pywb -v --doctest-modules ./pywb/ tests/
pytest --cov-config .coveragerc --cov pywb -v --doctest-modules ./pywb/ tests/