backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

New option --subdir-prefix

Browse Source 
Save WARCs in subdirectories equal to the current value of Warcprox-Meta['warc-prefix'].
E.g. if warc-prefix=='spn2' and --dir=/warcs, save them in /warcs/spn2/.
This commit is contained in:
Vangelis Banos 2024-06-03 21:21:19 +00:00
parent af52dec469
commit 56e0b17dc9
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warcprox/main.py
View File

@ -91,6 +91,8 @@ def _build_arg_parser(prog='warcprox', show_hidden=False):
help='where to store and load generated certificates')
arg_parser.add_argument('-d', '--dir', dest='directory',
default='./warcs', help='where to write warcs')
arg_parser.add_argument('--subdir-prefix', dest='subdir_prefix', action='store_true',
help='write warcs to --dir subdir equal to the current warc-prefix'),
arg_parser.add_argument('--warc-filename', dest='warc_filename',
default='{prefix}-{timestamp17}-{serialno}-{randomtoken}',
help='define custom WARC filename with variables {prefix}, {timestamp14}, {timestamp17}, {serialno}, {randomtoken}, {hostname}, {shorthostname}, {port}')

5
warcprox/writer.py
View File

@ -55,7 +55,10 @@ class WarcWriter:
self.open_suffix = '' if options.no_warc_open_suffix else '.open'
self.rollover_size = options.rollover_size or 1000000000
self.rollover_idle_time = options.rollover_idle_time or None
self.directory = options.directory or './warcs'
if options.subdir_prefix and options.prefix:
self.directory = os.path.sep.join([options.directory, options.prefix]) or './warcs'
else:
self.directory = options.directory or './warcs'
self.filename_template = options.warc_filename or \
'{prefix}-{timestamp17}-{randomtoken}-{serialno}'
self.last_activity = time.time()