backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

use locking to ensure consistency and avoid this kind of test failure https://travis-ci.org/internetarchive/warcprox/jobs/235819316

Browse Source
This commit is contained in:
Noah Levitt 2017-05-25 17:38:20 +00:00
parent 00b982aa24
commit 2f93cdcad9
4 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -50,7 +50,7 @@ except:
setuptools.setup(
name='warcprox',
version='2.1b1.dev86',
version='2.1b1.dev87',
description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox',
author='Noah Levitt',

3
warcprox/dedup.py
View File

@ -28,7 +28,6 @@ from hanzo import warctools
import warcprox
import random
import sqlite3
import threading
class DedupDb(object):
logger = logging.getLogger("warcprox.dedup.DedupDb")
@ -77,7 +76,7 @@ class DedupDb(object):
conn = sqlite3.connect(self.file)
conn.execute(
'insert into dedup (key, value) values (?, ?);',
'insert or replace into dedup (key, value) values (?, ?)',
(key, json_value))
conn.commit()
conn.close()

13
warcprox/playback.py
View File

@ -41,6 +41,7 @@ import re
from warcprox.mitmproxy import MitmProxyHandler
import warcprox
import sqlite3
import threading
class PlaybackProxyHandler(MitmProxyHandler):
logger = logging.getLogger("warcprox.playback.PlaybackProxyHandler")
@ -232,6 +233,7 @@ class PlaybackIndexDb(object):
def __init__(self, file='./warcprox.sqlite'):
self.file = file
self._lock = threading.RLock()
if os.path.exists(self.file):
self.logger.info(
@ -270,6 +272,7 @@ class PlaybackIndexDb(object):
# url:{date1:[record1={'f':warcfile,'o':response_offset,'q':request_offset,'i':record_id},record2,...],date2:[{...}],...}
with self._lock:
conn = sqlite3.connect(self.file)
cursor = conn.execute(
'select value from playback where url = ?', (url,))
@ -280,15 +283,17 @@ class PlaybackIndexDb(object):
py_value = {}
if date_str in py_value:
py_value[date_str].append({'f':warcfile, 'o':offset, 'i':record_id_str})
py_value[date_str].append(
{'f':warcfile, 'o':offset, 'i':record_id_str})
else:
py_value[date_str] = [{'f':warcfile, 'o':offset, 'i':record_id_str}]
py_value[date_str] = [
{'f':warcfile, 'o':offset, 'i':record_id_str}]
json_value = json.dumps(py_value, separators=(',',':'))
conn.execute(
'insert or replace into playback (url, value) values (?, ?)',
(url, json_value))
'insert or replace into playback (url, value) '
'values (?, ?)', (url, json_value))
conn.commit()
conn.close()

15
warcprox/stats.py
View File

@ -56,8 +56,10 @@ class StatsDb:
def __init__(self, file='./warcprox.sqlite', options=warcprox.Options()):
self.file = file
self.options = options
self._lock = threading.RLock()
def start(self):
with self._lock:
if os.path.exists(self.file):
self.logger.info(
'opening existing stats database %s', self.file)
@ -150,20 +152,13 @@ class StatsDb:
return buckets
def tally(self, recorded_url, records):
with self._lock:
conn = sqlite3.connect(self.file)
i = 0
for bucket in self.buckets(recorded_url):
try:
cursor = conn.execute(
'select stats from buckets_of_stats where bucket=?',
(bucket,))
except:
logging.info(
'i=%s bucket=%s self.file=%s', i, repr(bucket),
repr(self.file), exc_info=1)
raise
i += 1
result_tuple = cursor.fetchone()
cursor.close()
@ -184,8 +179,8 @@ class StatsDb:
json_value = json.dumps(bucket_stats, separators=(',',':'))
conn.execute(
'insert or replace into buckets_of_stats(bucket, stats) '
'values (?, ?)', (bucket, json_value))
'insert or replace into buckets_of_stats '
'(bucket, stats) values (?, ?)', (bucket, json_value))
conn.commit()
conn.close()