backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

timeouts for trough requests to prevent hanging

Browse Source
This commit is contained in:
Noah Levitt 2017-12-27 16:32:54 -08:00
parent eacf070a2a
commit a85c665ce9
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -52,7 +52,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.3.1b4.dev131', version='2.3.1b4.dev132',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

12
warcprox/trough.py
View File

@ -87,7 +87,7 @@ class TroughClient(object):
def promote(self, segment_id): def promote(self, segment_id):
url = os.path.join(self.segment_manager_url(), 'promote') url = os.path.join(self.segment_manager_url(), 'promote')
payload_dict = {'segment': segment_id} payload_dict = {'segment': segment_id}
response = requests.post(url, json=payload_dict) response = requests.post(url, json=payload_dict, timeout=21600)
if response.status_code != 200: if response.status_code != 200:
raise Exception( raise Exception(
'Received %s: %r in response to POST %s with data %s' % ( 'Received %s: %r in response to POST %s with data %s' % (
@ -129,7 +129,7 @@ class TroughClient(object):
def write_url_nocache(self, segment_id, schema_id='default'): def write_url_nocache(self, segment_id, schema_id='default'):
provision_url = os.path.join(self.segment_manager_url(), 'provision') provision_url = os.path.join(self.segment_manager_url(), 'provision')
payload_dict = {'segment': segment_id, 'schema': schema_id} payload_dict = {'segment': segment_id, 'schema': schema_id}
response = requests.post(provision_url, json=payload_dict) response = requests.post(provision_url, json=payload_dict, timeout=600)
if response.status_code != 200: if response.status_code != 200:
raise Exception( raise Exception(
'Received %s: %r in response to POST %s with data %s' % ( 'Received %s: %r in response to POST %s with data %s' % (
@ -175,7 +175,7 @@ class TroughClient(object):
sql = sql_tmpl % tuple(self.sql_value(v) for v in values) sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
try: try:
response = requests.post(write_url, sql) response = requests.post(write_url, sql, timeout=600)
if segment_id not in self._dirty_segments: if segment_id not in self._dirty_segments:
with self._dirty_segments_lock: with self._dirty_segments_lock:
self._dirty_segments.add(segment_id) self._dirty_segments.add(segment_id)
@ -200,7 +200,7 @@ class TroughClient(object):
return None return None
sql = sql_tmpl % tuple(self.sql_value(v) for v in values) sql = sql_tmpl % tuple(self.sql_value(v) for v in values)
try: try:
response = requests.post(read_url, sql) response = requests.post(read_url, sql, timeout=600)
except: except:
self._read_url_cache.pop(segment_id, None) self._read_url_cache.pop(segment_id, None)
self.logger.error( self.logger.error(
@ -221,7 +221,7 @@ class TroughClient(object):
def schema_exists(self, schema_id): def schema_exists(self, schema_id):
url = os.path.join(self.segment_manager_url(), 'schema', schema_id) url = os.path.join(self.segment_manager_url(), 'schema', schema_id)
response = requests.get(url) response = requests.get(url, timeout=60)
if response.status_code == 200: if response.status_code == 200:
return True return True
elif response.status_code == 404: elif response.status_code == 404:
@ -232,7 +232,7 @@ class TroughClient(object):
def register_schema(self, schema_id, sql): def register_schema(self, schema_id, sql):
url = os.path.join( url = os.path.join(
self.segment_manager_url(), 'schema', schema_id, 'sql') self.segment_manager_url(), 'schema', schema_id, 'sql')
response = requests.put(url, sql) response = requests.put(url, sql, timeout=600)
if response.status_code not in (201, 204): if response.status_code not in (201, 204):
raise Exception( raise Exception(
'Received %s: %r in response to PUT %r with data %r' % ( 'Received %s: %r in response to PUT %r with data %r' % (