backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Rename writer var and add exception handling

Browse Source 
Rename ``self._f_finalname_suffix`` to ``self._f_open_suffix``.

Add exception handling for file locking operations.
This commit is contained in:
vbanos 2017-10-27 16:22:16 +03:00
parent 975f2479a8
commit 5871a1bae2
1 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
warcprox/writer.py
View File

@ -54,7 +54,7 @@ class WarcWriter:
self._f = None
self._fpath = None
self._f_finalname = None
self._f_finalname_suffix = '' if options.no_warc_open_suffix else '.open'
self._f_open_suffix = '' if options.no_warc_open_suffix else '.open'
self._serial = 0
self._lock = threading.RLock()
@ -72,8 +72,12 @@ class WarcWriter:
with self._lock:
if self._fpath:
self.logger.info('closing %s', self._f_finalname)
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_UN)
if self._f_open_suffix == '':
try:
fcntl.flock(self._f, fcntl.LOCK_UN)
except IOError as exc:
self.logger.error('could not unlock file %s (%s)',
self._fpath, exc)
self._f.close()
finalpath = os.path.sep.join(
[self.directory, self._f_finalname])
@ -95,13 +99,17 @@ class WarcWriter:
self.prefix, self.timestamp17(), self._serial,
self._randomtoken, '.gz' if self.gzip else '')
self._fpath = os.path.sep.join([
self.directory, self._f_finalname + self._f_finalname_suffix])
self.directory, self._f_finalname + self._f_open_suffix])
self._f = open(self._fpath, 'wb')
# if no '.open' suffix is used for WARC, acquire an exclusive
# file lock.
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
if self._f_open_suffix == '':
try:
fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as exc:
self.logger.error('could not lock file %s (%s)',
self._fpath, exc)
warcinfo_record = self.record_builder.build_warcinfo_record(
self._f_finalname)