backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-24 06:59:52 +01:00
Code Issues Projects Releases Activity

html rewrite: add trailing slash for <base> tag rewrite if url is a scheme://host

Browse Source 
with no path component #77
cleanup: remove unused code path for tags with no rewriting -- all tags
now checked for dynamic attrs which may need rewriting
update tests, including live rewrite test dependent on live site (FB)
This commit is contained in:
Ilya Kreymer 2015-03-13 10:53:57 -07:00
parent 1fb631870b
commit 24021fcd57
3 changed files with 31 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pywb/rewrite/html_rewriter.py
View File

@ -5,6 +5,7 @@ import sys
import re import re
from HTMLParser import HTMLParser, HTMLParseError from HTMLParser import HTMLParser, HTMLParseError
from urlparse import urlsplit
from url_rewriter import UrlRewriter from url_rewriter import UrlRewriter
from regex_rewriters import JSRewriter, CSSRewriter from regex_rewriters import JSRewriter, CSSRewriter
@ -121,7 +122,22 @@ class HTMLRewriterMixin(object):
meta_refresh[m.end(1):]) meta_refresh[m.end(1):])
return meta_refresh return meta_refresh
# ===========================
def _rewrite_base(self, value, mod=''):
if not value.endswith('/'):
# check if hostname with no path,
# eg http://example.com
if not urlsplit(value).path:
value += '/'
base_value = self._rewrite_url(value, mod)
if self.opts.get('rewrite_base', True):
value = base_value
self.url_rewriter = (self.url_rewriter.
rebase_rewriter(base_value))
return value
def _rewrite_url(self, value, mod=None): def _rewrite_url(self, value, mod=None):
if value: if value:
@ -221,12 +237,7 @@ class HTMLRewriterMixin(object):
# special case: base tag # special case: base tag
elif (tag == 'base') and (attr_name == 'href') and attr_value: elif (tag == 'base') and (attr_name == 'href') and attr_value:
rw_mod = handler.get(attr_name) rw_mod = handler.get(attr_name)
base_value = self._rewrite_url(attr_value, rw_mod) attr_value = self._rewrite_base(attr_value, rw_mod)
if self.opts.get('rewrite_base', True):
attr_value = base_value
self.url_rewriter = (self.url_rewriter.
rebase_rewriter(base_value))
else: else:
# rewrite url using tag handler # rewrite url using tag handler
rw_mod = handler.get(attr_name) rw_mod = handler.get(attr_name)
@ -338,15 +349,15 @@ class HTMLRewriter(HTMLRewriterMixin, HTMLParser):
return s return s
def handle_starttag(self, tag, attrs): def handle_starttag(self, tag, attrs):
if not self._rewrite_tag_attrs(tag, attrs): self._rewrite_tag_attrs(tag, attrs)
self.out.write(self.get_starttag_text())
elif tag != 'head' or not self._rewrite_head(False): if tag != 'head' or not self._rewrite_head(False):
self.out.write('>') self.out.write('>')
def handle_startendtag(self, tag, attrs): def handle_startendtag(self, tag, attrs):
if not self._rewrite_tag_attrs(tag, attrs): self._rewrite_tag_attrs(tag, attrs)
self.out.write(self.get_starttag_text())
elif tag != 'head' or not self._rewrite_head(True): if tag != 'head' or not self._rewrite_head(True):
self.out.write('/>') self.out.write('/>')
def handle_endtag(self, tag): def handle_endtag(self, tag):

4
pywb/rewrite/test/test_html_rewriter.py
View File

@ -27,6 +27,10 @@ ur"""
>>> parse('<base href="static/"/><img src="image.gif"/>') >>> parse('<base href="static/"/><img src="image.gif"/>')
<base href="/web/20131226101010/http://example.com/some/path/static/"/><img src="/web/20131226101010im_/http://example.com/some/path/static/image.gif"/> <base href="/web/20131226101010/http://example.com/some/path/static/"/><img src="/web/20131226101010im_/http://example.com/some/path/static/image.gif"/>
# ensure trailing slash added
>>> parse('<base href="http://example.com"/>')
<base href="/web/20131226101010/http://example.com/"/>
# Base Tests -- no rewrite # Base Tests -- no rewrite
>>> parse('<html><head><base href="http://example.com/diff/path/file.html"/>', urlrewriter=no_base_canon_rewriter) >>> parse('<html><head><base href="http://example.com/diff/path/file.html"/>', urlrewriter=no_base_canon_rewriter)
<html><head><base href="http://example.com/diff/path/file.html"/> <html><head><base href="http://example.com/diff/path/file.html"/>

5
pywb/rewrite/test/test_rewrite_live.py
View File

@ -203,8 +203,9 @@ def test_example_4_rewrite_err():
def test_example_domain_specific_3(): def test_example_domain_specific_3():
status_headers, buff = get_rewritten('http://facebook.com/digitalpreservation', urlrewriter, follow_redirects=True) status_headers, buff = get_rewritten('http://facebook.com/digitalpreservation', urlrewriter, follow_redirects=True)
# comment out bootloader # comment out Bootloader.configurePage, if it is still there
assert '/* Bootloader.configurePage' in buff if 'Bootloader.configurePage' in buff:
assert '/* Bootloader.configurePage' in buff
def test_wombat_top(): def test_wombat_top():
#status_headers, buff = get_rewritten('https://assets-cdn.github.com/assets/github-0f06d0f46fe7bcfbf31f2380f23aec15ba21b8ec.js', urlrewriter) #status_headers, buff = get_rewritten('https://assets-cdn.github.com/assets/github-0f06d0f46fe7bcfbf31f2380f23aec15ba21b8ec.js', urlrewriter)