backup/pywb
1
0
Fork 0
mirror of https://github.com/webrecorder/pywb.git synced 2025-03-15 00:03:28 +01:00
Code Issues Projects Releases Activity

[#795] wb-manager: Show error when adding duplicate warc files (#797)

Browse Source
This commit is contained in:
kuechensofa 2023-01-19 16:26:56 +00:00 committed by GitHub
parent 1790fd006a
commit e6ec8b4aeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pywb/manager/manager.py
View File

@ -121,14 +121,24 @@ directory structure expected by pywb
format(self.archive_dir)) format(self.archive_dir))
full_paths = [] full_paths = []
duplicate_warcs = []
for filename in warcs: for filename in warcs:
filename = os.path.abspath(filename) filename = os.path.abspath(filename)
# don't overwrite existing warcs with duplicate names
if os.path.exists(os.path.join(self.archive_dir, os.path.basename(filename))):
duplicate_warcs.append(filename)
continue
shutil.copy2(filename, self.archive_dir) shutil.copy2(filename, self.archive_dir)
full_paths.append(os.path.join(self.archive_dir, filename)) full_paths.append(os.path.join(self.archive_dir, filename))
logging.info('Copied ' + filename + ' to ' + self.archive_dir) logging.info('Copied ' + filename + ' to ' + self.archive_dir)
self._index_merge_warcs(full_paths, self.DEF_INDEX_FILE) self._index_merge_warcs(full_paths, self.DEF_INDEX_FILE)
if duplicate_warcs:
logging.warning(f'Warcs {", ".join(duplicate_warcs)} weren\'t added because of duplicate names.')
def reindex(self): def reindex(self):
cdx_file = os.path.join(self.indexes_dir, self.DEF_INDEX_FILE) cdx_file = os.path.join(self.indexes_dir, self.DEF_INDEX_FILE)
logging.info('Indexing ' + self.archive_dir + ' to ' + cdx_file) logging.info('Indexing ' + self.archive_dir + ' to ' + cdx_file)