backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Acquire and exclusive file lock when not using .open WARC suffix

Browse Source
This commit is contained in:
Vangelis Banos 2017-10-26 21:58:31 +00:00
parent c9f1feb3db
commit 975f2479a8
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
warcprox/writer.py
View File

@ -24,6 +24,7 @@ from __future__ import absolute_import
import logging import logging
from datetime import datetime from datetime import datetime
from hanzo import warctools from hanzo import warctools
import fcntl
import time import time
import warcprox import warcprox
import os import os
@ -71,6 +72,8 @@ class WarcWriter:
with self._lock: with self._lock:
if self._fpath: if self._fpath:
self.logger.info('closing %s', self._f_finalname) self.logger.info('closing %s', self._f_finalname)
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_UN)
self._f.close() self._f.close()
finalpath = os.path.sep.join( finalpath = os.path.sep.join(
[self.directory, self._f_finalname]) [self.directory, self._f_finalname])
@ -95,6 +98,10 @@ class WarcWriter:
self.directory, self._f_finalname + self._f_finalname_suffix]) self.directory, self._f_finalname + self._f_finalname_suffix])
self._f = open(self._fpath, 'wb') self._f = open(self._fpath, 'wb')
# if no '.open' suffix is used for WARC, acquire an exclusive
# file lock.
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
warcinfo_record = self.record_builder.build_warcinfo_record( warcinfo_record = self.record_builder.build_warcinfo_record(
self._f_finalname) self._f_finalname)