backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Merge pull request #168 from internetarchive/adds-hop-path-logging

Browse Source 
Adds hop path logging
This commit is contained in:
Adam Miller 2022-02-09 10:55:12 -08:00 committed by GitHub
commit dea2d1c8fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -2,7 +2,7 @@
''' '''
setup.py - setuptools installation configuration for warcprox setup.py - setuptools installation configuration for warcprox
Copyright (C) 2013-2020 Internet Archive Copyright (C) 2013-2021 Internet Archive
This program is free software; you can redistribute it and/or This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License modify it under the terms of the GNU General Public License

7
warcprox/crawl_log.py
View File

@ -64,12 +64,17 @@ class CrawlLogger(object):
else: else:
content_length = 0 content_length = 0
payload_digest = '-' payload_digest = '-'
logging.info('metadata %s', recorded_url.warcprox_meta)
hop_path = recorded_url.warcprox_meta["hop_path"] if "hop_path" in recorded_url.warcprox_meta and recorded_url.warcprox_meta["hop_path"] and len(recorded_url.warcprox_meta["hop_path"].strip()) > 0 else '-'
hop_path_parent = recorded_url.warcprox_meta["hop_path_parent"] if "hop_path_parent" in recorded_url.warcprox_meta else None
if hop_path_parent and hop_path_parent != recorded_url.url:
hop_path = str(hop_path if hop_path and hop_path != "-" else "") + "B"
fields = [ fields = [
'{:%Y-%m-%dT%H:%M:%S}.{:03d}Z'.format(now, now.microsecond//1000), '{:%Y-%m-%dT%H:%M:%S}.{:03d}Z'.format(now, now.microsecond//1000),
'% 5s' % status, '% 5s' % status,
'% 10s' % content_length, '% 10s' % content_length,
recorded_url.url, recorded_url.url,
'-', # hop path hop_path,
recorded_url.referer or '-', recorded_url.referer or '-',
recorded_url.mimetype if recorded_url.mimetype is not None and recorded_url.mimetype.strip() else '-', recorded_url.mimetype if recorded_url.mimetype is not None and recorded_url.mimetype.strip() else '-',
'-', '-',