From 975f2479a8caa5f5a8e0e0328b56c956b6563bbb Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Thu, 26 Oct 2017 21:58:31 +0000 Subject: [PATCH] Acquire and exclusive file lock when not using .open WARC suffix --- warcprox/writer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/warcprox/writer.py b/warcprox/writer.py index 419fd77..7e7ff11 100644 --- a/warcprox/writer.py +++ b/warcprox/writer.py @@ -24,6 +24,7 @@ from __future__ import absolute_import import logging from datetime import datetime from hanzo import warctools +import fcntl import time import warcprox import os @@ -71,6 +72,8 @@ class WarcWriter: with self._lock: if self._fpath: self.logger.info('closing %s', self._f_finalname) + if self._f_finalname_suffix == '': + fcntl.flock(self._f, fcntl.LOCK_UN) self._f.close() finalpath = os.path.sep.join( [self.directory, self._f_finalname]) @@ -95,6 +98,10 @@ class WarcWriter: self.directory, self._f_finalname + self._f_finalname_suffix]) self._f = open(self._fpath, 'wb') + # if no '.open' suffix is used for WARC, acquire an exclusive + # file lock. + if self._f_finalname_suffix == '': + fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB) warcinfo_record = self.record_builder.build_warcinfo_record( self._f_finalname)