From a85c665ce914106b2077a3a34bfc0ab325642674 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 27 Dec 2017 16:32:54 -0800 Subject: [PATCH] timeouts for trough requests to prevent hanging --- setup.py | 2 +- warcprox/trough.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 65047ef..86fdc12 100755 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ except: setuptools.setup( name='warcprox', - version='2.3.1b4.dev131', + version='2.3.1b4.dev132', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/trough.py b/warcprox/trough.py index 76aafed..97912b0 100644 --- a/warcprox/trough.py +++ b/warcprox/trough.py @@ -87,7 +87,7 @@ class TroughClient(object): def promote(self, segment_id): url = os.path.join(self.segment_manager_url(), 'promote') payload_dict = {'segment': segment_id} - response = requests.post(url, json=payload_dict) + response = requests.post(url, json=payload_dict, timeout=21600) if response.status_code != 200: raise Exception( 'Received %s: %r in response to POST %s with data %s' % ( @@ -129,7 +129,7 @@ class TroughClient(object): def write_url_nocache(self, segment_id, schema_id='default'): provision_url = os.path.join(self.segment_manager_url(), 'provision') payload_dict = {'segment': segment_id, 'schema': schema_id} - response = requests.post(provision_url, json=payload_dict) + response = requests.post(provision_url, json=payload_dict, timeout=600) if response.status_code != 200: raise Exception( 'Received %s: %r in response to POST %s with data %s' % ( @@ -175,7 +175,7 @@ class TroughClient(object): sql = sql_tmpl % tuple(self.sql_value(v) for v in values) try: - response = requests.post(write_url, sql) + response = requests.post(write_url, sql, timeout=600) if segment_id not in self._dirty_segments: with self._dirty_segments_lock: self._dirty_segments.add(segment_id) @@ -200,7 +200,7 @@ class TroughClient(object): return None sql = sql_tmpl % tuple(self.sql_value(v) for v in values) try: - response = requests.post(read_url, sql) + response = requests.post(read_url, sql, timeout=600) except: self._read_url_cache.pop(segment_id, None) self.logger.error( @@ -221,7 +221,7 @@ class TroughClient(object): def schema_exists(self, schema_id): url = os.path.join(self.segment_manager_url(), 'schema', schema_id) - response = requests.get(url) + response = requests.get(url, timeout=60) if response.status_code == 200: return True elif response.status_code == 404: @@ -232,7 +232,7 @@ class TroughClient(object): def register_schema(self, schema_id, sql): url = os.path.join( self.segment_manager_url(), 'schema', schema_id, 'sql') - response = requests.put(url, sql) + response = requests.put(url, sql, timeout=600) if response.status_code not in (201, 204): raise Exception( 'Received %s: %r in response to PUT %r with data %r' % (