backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

Acquire and exclusive file lock when not using .open WARC suffix

Browse Source
This commit is contained in:
Vangelis Banos 2017-10-26 21:58:31 +00:00
parent c9f1feb3db
commit 975f2479a8
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
warcprox/writer.py
View File

@ -24,6 +24,7 @@ from __future__ import absolute_import
import logging
from datetime import datetime
from hanzo import warctools
import fcntl
import time
import warcprox
import os
@ -71,6 +72,8 @@ class WarcWriter:
with self._lock:
if self._fpath:
self.logger.info('closing %s', self._f_finalname)
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_UN)
self._f.close()
finalpath = os.path.sep.join(
[self.directory, self._f_finalname])
@ -95,6 +98,10 @@ class WarcWriter:
self.directory, self._f_finalname + self._f_finalname_suffix])
self._f = open(self._fpath, 'wb')
# if no '.open' suffix is used for WARC, acquire an exclusive
# file lock.
if self._f_finalname_suffix == '':
fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
warcinfo_record = self.record_builder.build_warcinfo_record(
self._f_finalname)