backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Fixing referer on request with null hop path

Browse Source
This commit is contained in:
Adam Miller 2022-03-04 02:01:55 +00:00
parent 3a234d0cec
commit ade2373711
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
warcprox/crawl_log.py
View File

@ -68,7 +68,7 @@ class CrawlLogger(object):
hop_path = recorded_url.warcprox_meta.get('metadata', {}).get('hop_path', '-') hop_path = recorded_url.warcprox_meta.get('metadata', {}).get('hop_path', '-')
if hop_path is None: if hop_path is None:
hop_path = "-" hop_path = "-"
hop_path_referer = recorded_url.warcprox_meta.get('metadata', {}).get('hop_path_referer', recorded_url.referer) hop_path_referer = recorded_url.warcprox_meta.get('metadata', {}).get('hop_path_referer', "-")
if hop_path_referer != recorded_url.url.decode('ascii'): if hop_path_referer != recorded_url.url.decode('ascii'):
if hop_path == "-": if hop_path == "-":
hop_path = "B" hop_path = "B"
@ -80,7 +80,7 @@ class CrawlLogger(object):
'% 10s' % content_length, '% 10s' % content_length,
recorded_url.url, recorded_url.url,
hop_path, hop_path,
recorded_url.referer or hop_path_referer or '-', recorded_url.referer or hop_path_referer if hop_path != "-" else "-",
recorded_url.mimetype if recorded_url.mimetype is not None and recorded_url.mimetype.strip() else '-', recorded_url.mimetype if recorded_url.mimetype is not None and recorded_url.mimetype.strip() else '-',
'-', '-',
'{:%Y%m%d%H%M%S}{:03d}+{:03d}'.format( '{:%Y%m%d%H%M%S}{:03d}+{:03d}'.format(