backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

htmlrewriter: if urls contain non-ascii chars, ensure the url is reencoded with expected charset, using same charset as for banner insert (#361)

Browse Source 
(instead of default iso-8859-1) before %-encoding and rewriting
tests: add test to ensure correct %-encoding of utf-8 urls
This commit is contained in:
Ilya Kreymer 2018-08-06 22:42:24 -07:00 committed by GitHub
parent 1156032e0e
commit 5476d75294
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 34 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pywb/rewrite/content_rewriter.py
View File

@ -130,31 +130,31 @@ class BaseContentRewriter(object):
head_insert=head_insert_str,
url=cdx['url'],
defmod=self.replay_mod,
parse_comments=rule.get('parse_comments', False))
parse_comments=rule.get('parse_comments', False),
charset=rwinfo.charset)
return rw
def get_head_insert(self, rwinfo, rule, head_insert_func, cdx):
head_insert_str = ''
charset = rwinfo.charset
# if no charset set, attempt to extract from first 1024
if not charset:
if not rwinfo.charset:
first_buff = rwinfo.read_and_keep(1024)
charset = self.extract_html_charset(first_buff)
rwinfo.charset = self.extract_html_charset(first_buff)
if head_insert_func:
head_insert_orig = head_insert_func(rule, cdx)
if charset:
if rwinfo.charset:
try:
head_insert_str = webencodings.encode(head_insert_orig, charset)
head_insert_str = webencodings.encode(head_insert_orig, rwinfo.charset)
except:
pass
if not head_insert_str:
charset = 'utf-8'
head_insert_str = head_insert_orig.encode(charset)
rwinfo.charset = 'utf-8'
head_insert_str = head_insert_orig.encode(rwinfo.charset)
head_insert_str = head_insert_str.decode('iso-8859-1')

13
pywb/rewrite/html_rewriter.py
View File

@ -115,9 +115,11 @@ class HTMLRewriterMixin(StreamingRewriter):
css_rewriter_class=None,
url = '',
defmod='',
parse_comments=False):
parse_comments=False,
charset='utf-8'):
super(HTMLRewriterMixin, self).__init__(url_rewriter, False)
self.charset = charset
self._wb_parse_context = None
if js_rewriter:
@ -229,6 +231,15 @@ class HTMLRewriterMixin(StreamingRewriter):
if not value:
return ''
# if url is not ascii, ensure its reencoded in expected charset
try:
value.encode('ascii')
except:
try:
value = value.encode('iso-8859-1').decode(self.charset)
except:
pass
unesc_value = self.try_unescape(value)
rewritten_value = self.url_rewriter.rewrite(unesc_value, mod, force_abs)

14
pywb/rewrite/test/test_content_rewriter.py
View File

@ -1,3 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from warcio.warcwriter import BufferWARCWriter, GzippingWrapper
from warcio.statusandheaders import StatusAndHeaders
@ -105,6 +108,17 @@ class TestContentRewriter(object):
assert ('Content-Type', 'text/html; charset=UTF-8') in headers.headers
assert b''.join(gen).decode('utf-8') == exp
def test_rewrite_html_utf_8(self):
headers = {'Content-Type': 'text/html; charset=utf-8'}
content = u'<html><body><a href="http://éxample.com/tésté"></a></body></html>'
headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701mp_')
exp = '<html><body><a href="http://localhost:8080/prefix/201701/http://%C3%A9xample.com/t%C3%A9st%C3%A9"></a></body></html>'
assert is_rw
assert ('Content-Type', 'text/html; charset=utf-8') in headers.headers
assert b''.join(gen).decode('utf-8') == exp
def test_rewrite_html_js_mod(self, headers):
content = '<html><body><a href="http://example.com/"></a></body></html>'