From 504af2fb0f2f45633adda8f8a80edf480915cdff Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 7 Sep 2016 13:01:11 -0700 Subject: [PATCH] try to avoid ever blocking when sending messages to kafka --- README.rst | 1 - setup.py | 2 +- warcprox/kafkafeed.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 1a4bfbb..388bcf5 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,6 @@ Warcprox runs on python 3.4. To install latest release run: - :: # apt-get install libffi-dev libssl-dev python3-gdbm diff --git a/setup.py b/setup.py index 844d8ca..147cb35 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ except: setuptools.setup( name='warcprox', - version='2.0b2.dev26', + version='2.0b2.dev27', description='WARC writing MITM HTTP/S proxy', url='https://github.com/internetarchive/warcprox', author='Noah Levitt', diff --git a/warcprox/kafkafeed.py b/warcprox/kafkafeed.py index 8b2dbf9..683e925 100644 --- a/warcprox/kafkafeed.py +++ b/warcprox/kafkafeed.py @@ -32,7 +32,9 @@ class CaptureFeed: def __init__(self, broker_list, topic=None): self.broker_list = broker_list self.topic = topic - self._producer = kafka.KafkaProducer(bootstrap_servers=broker_list) + # acks=0 to avoid ever blocking + self._producer = kafka.KafkaProducer( + bootstrap_servers=broker_list, acks=0) def notify(self, recorded_url, records): if records[0].type not in (b'revisit', b'response'):