mirror of
https://github.com/webrecorder/pywb.git
synced 2025-03-20 02:39:13 +01:00
372 lines
8.4 KiB
YAML
372 lines
8.4 KiB
YAML
rules:
|
|
|
|
|
|
# twitter rules
|
|
#=================================================================
|
|
- url_prefix: 'com,twitter)/i/profiles/show/'
|
|
|
|
fuzzy_lookup: '/profiles/show/.*with_replies\?.*(max_id=[^&]+)'
|
|
|
|
- url_prefix: 'com,twitter)/i/timeline'
|
|
|
|
fuzzy_lookup:
|
|
- max_position
|
|
- include_entities
|
|
|
|
- url_prefix: 'com,twitter)/i/videos/tweet'
|
|
|
|
fuzzy_lookup: '()'
|
|
|
|
|
|
# facebook rules
|
|
#=================================================================
|
|
- url_prefix: 'com,facebook)/ajax/pagelet/generic.php/'
|
|
|
|
fuzzy_lookup: 'com,facebook\)/.*[?&]data=(.*?(?:[&]|query_type[^,]+))'
|
|
|
|
- url_prefix: 'com,facebook)/ajax/ufi/'
|
|
|
|
fuzzy_lookup:
|
|
- ft_ent_identifier
|
|
- lsd
|
|
|
|
- url_prefix: 'com,facebook)/ajax/chat/hovercard/sidebar.php'
|
|
|
|
fuzzy_lookup:
|
|
- ids[0]
|
|
|
|
- url_prefix: 'com,facebook)/login.php'
|
|
|
|
fuzzy_lookup:
|
|
- email
|
|
- lgnrnd
|
|
- lsd
|
|
|
|
- url_prefix: 'com,facebook)/ajax/timezone/update.php'
|
|
|
|
fuzzy_lookup:
|
|
- __user
|
|
|
|
# fallback for all /ajax/
|
|
- url_prefix: 'com,facebook)/ajax/'
|
|
|
|
fuzzy_lookup: '([?&][^_]\w+=[^&]+)+'
|
|
|
|
- url_prefix: 'com,facebook)/'
|
|
rewrite:
|
|
js_regexs:
|
|
- match: 'Bootloader\.configurePage.*?;'
|
|
replace: '/* {0} */'
|
|
|
|
- match: 'dash_manifest:"(.*",dash_prefetched_representation_ids:.*?])'
|
|
group: 1
|
|
function: 'pywb.rewrite.rewrite_dash:rewrite_fb_dash'
|
|
|
|
parse_comments: true
|
|
|
|
- url_prefix: 'com,facebook'
|
|
rewrite:
|
|
cookie_scope: root
|
|
|
|
|
|
# fastly
|
|
#=================================================================
|
|
- url_prefix: ['net,fastly,']
|
|
|
|
fuzzy_lookup: '()'
|
|
|
|
|
|
# instagram rules
|
|
#=================================================================
|
|
- url_prefix: 'net,cloudfront,'
|
|
|
|
rewrite:
|
|
js_regexs:
|
|
- match: '\burl\((//[^)]+)\)'
|
|
rewrite: true
|
|
group: 1
|
|
|
|
- url_prefix: 'com,instagram)/'
|
|
|
|
fuzzy_lookup: '()'
|
|
|
|
|
|
# blogspot rules
|
|
#=================================================================
|
|
- url_prefix: ['com,blogspot,']
|
|
|
|
fuzzy_lookup:
|
|
match:
|
|
- path
|
|
- action
|
|
- widgettype
|
|
|
|
replace: '.com/'
|
|
|
|
|
|
# flickr rules
|
|
#=================================================================
|
|
- url_prefix: ['com,yimg,l)/g/combo', 'com,yimg,s)/pw/combo', 'com,yahooapis,yui)/combo']
|
|
fuzzy_lookup: '([^/]+(?:\.css|\.js))'
|
|
|
|
|
|
- url_prefix: 'com,staticflickr,'
|
|
|
|
fuzzy_lookup:
|
|
match: '([0-9]+_[a-z0-9]+).*?.jpg'
|
|
replace: '/'
|
|
|
|
|
|
# google plus rules
|
|
#=================================================================
|
|
|
|
- url_prefix: 'com,google,plus)/_/stream/getactivities'
|
|
|
|
# fuzzy_lookup: '(egk[^"]+)?.*(f.sid=[^&]+)'
|
|
fuzzy_lookup: 'f.req=.*\]\]\]\,\"([^"]+).*(f.sid=[^&]+)'
|
|
|
|
- url_prefix: 'com,google,plus)/_/stream/squarestream'
|
|
|
|
#fuzzy_lookup: '(cai[^"]+).*(f.sid=[^&]+)'
|
|
fuzzy_lookup: 'f.req=.*?\"([^"]+).*(f.sid=[^&]+)'
|
|
|
|
- url_prefix: 'com,google,plus)/_/communities/rt/landing'
|
|
|
|
fuzzy_lookup: 'com,google,plus\)/_/.*?.*\,(\d{13}\])&.*(f.sid=[^&]+).*'
|
|
|
|
|
|
- url_prefix: 'com,google,plus)/_/'
|
|
|
|
fuzzy_lookup: 'com,google,plus\)/_/.*?.*(f.sid=[^&]+)'
|
|
|
|
# periscope
|
|
#=================================================================
|
|
|
|
- url_prefix: 'tv,periscope,assets)/js/'
|
|
|
|
rewrite:
|
|
js_regexs:
|
|
- match: '"location"'
|
|
replace: '"WB_wombat_location"'
|
|
|
|
|
|
# medium
|
|
#=================================================================
|
|
|
|
- url_prefix: 'com,medium'
|
|
|
|
rewrite:
|
|
js_regexs:
|
|
- match: '"noitacol"'
|
|
replace: '"noitacol_tabmow_BW"'
|
|
|
|
|
|
# vimeo rules
|
|
#=================================================================
|
|
|
|
- url_prefix: 'com,vimeo,av)/'
|
|
|
|
# only use non query part of url, ignore query
|
|
fuzzy_lookup: '()'
|
|
|
|
- url_prefix: 'com,vimeocdn,'
|
|
|
|
fuzzy_lookup: '()'
|
|
|
|
- url_prefix: 'metadata)/player.vimeo.com/'
|
|
|
|
fuzzy_lookup:
|
|
match: '()'
|
|
replace: '&'
|
|
|
|
- url_prefix: 'com,vimeo,player)/log/'
|
|
|
|
fuzzy_lookup:
|
|
- id
|
|
- ownerId
|
|
- videoFileId
|
|
- signature
|
|
|
|
# vine
|
|
- url_prefix: 'co,vine,cdn,'
|
|
|
|
fuzzy_lookup:
|
|
replace: 'videos'
|
|
match: 'videos(?:_[^/]+)?/([^?]+)'
|
|
|
|
|
|
- url_prefix: 'com,disqus)/embed/comments'
|
|
|
|
fuzzy_lookup:
|
|
- base
|
|
- version
|
|
- t_i
|
|
|
|
|
|
|
|
# youtube rules
|
|
#=================================================================
|
|
|
|
- url_prefix: ['com,youtube)/get_video_info', 'com,youtube-nocookie)/get_video_info']
|
|
|
|
fuzzy_lookup:
|
|
- video_id
|
|
- html5
|
|
|
|
- url_prefix: 'com,youtube,s)/api/stats/qoe'
|
|
|
|
fuzzy_lookup:
|
|
- docid
|
|
|
|
- url_prefix: 'com,youtube,s)/api/stats/watch'
|
|
|
|
fuzzy_lookup:
|
|
- docid
|
|
|
|
- url_prefix: 'com,youtube,c'
|
|
|
|
fuzzy_lookup:
|
|
match:
|
|
regex: 'com,youtube,c.*/videogoodput.*'
|
|
args:
|
|
- id
|
|
|
|
# yt comments POST
|
|
- url_prefix: 'com,youtube)/watch_fragments_ajax'
|
|
|
|
fuzzy_lookup:
|
|
- v
|
|
- frags
|
|
|
|
- url_prefix: 'com,youtube)/comment_ajax'
|
|
|
|
fuzzy_lookup:
|
|
- video_id
|
|
- page_token
|
|
- action_load_comments
|
|
- filter
|
|
|
|
|
|
- url_prefix: 'com,googlevideo,'
|
|
|
|
fuzzy_lookup:
|
|
match:
|
|
regex: 'com,googlevideo.*/videoplayback.*'
|
|
args:
|
|
- id
|
|
- itag
|
|
#- mime
|
|
|
|
filter:
|
|
- '~urlkey:{0}'
|
|
- '!mimetype:text/plain'
|
|
|
|
type: 'domain'
|
|
|
|
# comments support
|
|
- url_prefix: 'com,googleapis,plus)/u/0/_/widget/render/comments'
|
|
|
|
rewrite:
|
|
js_rewrite_location: location
|
|
|
|
fuzzy_lookup:
|
|
- href
|
|
- stream_id
|
|
- substream_id
|
|
|
|
- url_prefix: 'com,ytimg,s)/yts/jsbin/html5player-new'
|
|
|
|
rewrite:
|
|
js_regexs:
|
|
- match: '\"0\"\=\=c\.dash'
|
|
replace: 'c.dash="0";c.dashmpd="";{0}'
|
|
|
|
# watch and embed config changes
|
|
- url_prefix: ['com,youtube)/', 'com,youtube-nocookie)/']
|
|
|
|
rewrite:
|
|
js_regexs:
|
|
- match: 'ytplayer.load\(\);'
|
|
replace: 'ytplayer.config.args.dash = "0"; ytplayer.config.args.dashmpd = ""; {0}'
|
|
|
|
- match: 'yt\.setConfig.*PLAYER_CONFIG.*args":\s*{'
|
|
replace: '{0} "dash": "0", dashmpd: "", '
|
|
|
|
req_cookie_rewrite:
|
|
- match: '^(((?!PREF).)*)$'
|
|
replace: '\1; PREF=f2=40000000'
|
|
|
|
- match: '(.*PREF=)([^ ;]*)(.*)'
|
|
replace: '\1&f2=40000000\3'
|
|
|
|
# testing rules -- not for valid domain
|
|
#=================================================================
|
|
# this rule block is a non-existent prefix merely for testing
|
|
|
|
- url_prefix: 'example,example,test,all)/'
|
|
|
|
rewrite:
|
|
js_rewrite_location: all
|
|
|
|
- url_prefix: 'example,example,test,loconly)/'
|
|
|
|
rewrite:
|
|
js_rewrite_location: location
|
|
|
|
- url_prefix: 'example,example,test,norewrite)/'
|
|
|
|
rewrite:
|
|
js_rewrite_location: none
|
|
|
|
- url_prefix: 'example,example,test)/'
|
|
|
|
canonicalize:
|
|
match: '(example,example,test\)/.*?)[?].*?(id=value).*'
|
|
replace: '\1?\2'
|
|
|
|
fuzzy_lookup:
|
|
- param1
|
|
- id
|
|
|
|
rewrite:
|
|
js_rewrite_location: urls
|
|
|
|
req_cookie_rewrite:
|
|
- match: '^(((?!FOO).)*)$'
|
|
replace: '\1; FOO=bar=1'
|
|
|
|
- match: '(.*FOO=)([^ ;]*)(.*)'
|
|
replace: '\1&bar=1\3'
|
|
|
|
- match: ''
|
|
invalid_: ''
|
|
|
|
|
|
# all domain rules -- fallback to this dataset
|
|
#=================================================================
|
|
# Applies to all urls -- should be last
|
|
- url_prefix: ''
|
|
fuzzy_lookup:
|
|
match: '()'
|
|
match_filters:
|
|
- not_ext:
|
|
- asp
|
|
- aspx
|
|
- jsp
|
|
- php
|
|
- pl
|
|
- exe
|
|
- dll
|
|
|
|
match: '()'
|
|
|
|
- mime: 'application/dash+xml'
|
|
match: '()'
|
|
|
|
- mime: 'application/x-shockwave-flash'
|
|
match: '()'
|
|
|
|
- mime: '*'
|
|
match: '(.*)[&?](?:_|cb|uncache)=[\d]+[&]?'
|
|
|