backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

Resolve relative canonical paths if rewriting is disabled

Browse Source 
For Via, we want rel=canonical links to resolve to the same
absolute URL as it did on the original page.

For absolute URLs, no rewriting is necessary. If the original
rel=canonical URL was relative however, it needs to be resolved
relative to the original URL.

See https://github.com/hypothesis/via/issues/65 for context.
This commit is contained in:
Robert Knight 2015-12-10 08:23:14 +00:00
parent 7a0680fb35
commit 83a33e0541
2 changed files with 31 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pywb/rewrite/html_rewriter.py
View File

@ -5,7 +5,7 @@ import sys
import re
from HTMLParser import HTMLParser, HTMLParseError
from urlparse import urlsplit, urlunsplit
from urlparse import urljoin, urlsplit, urlunsplit
from url_rewriter import UrlRewriter
from regex_rewriters import JSRewriter, CSSRewriter
@ -276,9 +276,18 @@ class HTMLRewriterMixin(object):
# special case: if rewrite_canon not set,
# don't rewrite rel=canonical
elif tag == 'link' and attr_name == 'href':
if (self.opts.get('rewrite_rel_canon', True) or
not self.has_attr(tag_attrs, ('rel', 'canonical'))):
rw_mod = handler.get(attr_name)
rw_mod = handler.get(attr_name)
if self.has_attr(tag_attrs, ('rel', 'canonical')):
if self.opts.get('rewrite_rel_canon', True):
attr_value = self._rewrite_url(attr_value, rw_mod)
else:
# resolve relative rel=canonical URLs so that they
# refer to the same absolute URL as on the original
# page (see https://github.com/hypothesis/via/issues/65
# for context)
attr_value = urljoin(self.orig_url, attr_value)
else:
attr_value = self._rewrite_url(attr_value, rw_mod)
# special case: meta tag

34
pywb/rewrite/test/test_html_rewriter.py
View File

@ -158,8 +158,12 @@ ur"""
<link rel="canonical" href="/web/20131226101010oe_/http://example.com/">
# rel=canonical: no_rewrite
>>> parse('<link rel=canonical href="http://example.com/">', urlrewriter=no_base_canon_rewriter)
<link rel="canonical" href="http://example.com/">
>>> parse('<link rel=canonical href="http://example.com/canon/path">', urlrewriter=no_base_canon_rewriter)
<link rel="canonical" href="http://example.com/canon/path">
# rel=canonical: no_rewrite
>>> parse('<link rel=canonical href="/relative/path">', urlrewriter=no_base_canon_rewriter)
<link rel="canonical" href="http://example.com/relative/path">
# doctype
>>> parse('<!doctype html PUBLIC "public">')
@ -210,26 +214,24 @@ from pywb.rewrite.html_rewriter import HTMLRewriter
import pprint
import urllib
urlrewriter = UrlRewriter('20131226101010/http://example.com/some/path/index.html',
'/web/',
rewrite_opts=dict(punycode_links=False))
ORIGINAL_URL = 'http://example.com/some/path/index.html'
full_path_urlrewriter = UrlRewriter('20131226101010/http://example.com/some/path/index.html',
'http://localhost:80/web/',
rewrite_opts=dict(punycode_links=False))
def new_rewriter(prefix='/web/', rewrite_opts=dict()):
PROXY_PATH = '20131226101010/{0}'.format(ORIGINAL_URL)
return UrlRewriter(PROXY_PATH, prefix, rewrite_opts=rewrite_opts)
urlrewriter_pencode = UrlRewriter('20131226101010/http://example.com/some/path/index.html',
'/web/',
rewrite_opts=dict(punycode_links=True))
urlrewriter = new_rewriter(rewrite_opts=dict(punycode_links=False))
full_path_urlrewriter = new_rewriter(prefix='http://localhost:80/web/',
rewrite_opts=dict(punycode_links=False))
no_base_canon_rewriter = UrlRewriter('20131226101010/http://example.com/some/path/index.html',
'/web/',
rewrite_opts=dict(rewrite_rel_canon=False,
rewrite_base=False))
urlrewriter_pencode = new_rewriter(rewrite_opts=dict(punycode_links=True))
no_base_canon_rewriter = new_rewriter(rewrite_opts=dict(rewrite_rel_canon=False,
rewrite_base=False))
def parse(data, head_insert=None, urlrewriter=urlrewriter):
parser = HTMLRewriter(urlrewriter, head_insert = head_insert)
parser = HTMLRewriter(urlrewriter, head_insert = head_insert, url = ORIGINAL_URL)
if isinstance(data, unicode):
data = data.encode('utf-8')