backup/warcprox
1
0
Fork 0
mirror of https://github.com/internetarchive/warcprox.git synced 2025-01-18 13:22:09 +01:00
Code Issues Projects Releases Activity

try to avoid ever blocking when sending messages to kafka

Browse Source
This commit is contained in:
Noah Levitt 2016-09-07 13:01:11 -07:00
parent 1ddebbc50e
commit 504af2fb0f
3 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.rst
View File

@ -13,7 +13,6 @@ Warcprox runs on python 3.4.
To install latest release run: To install latest release run:
:: ::
# apt-get install libffi-dev libssl-dev python3-gdbm # apt-get install libffi-dev libssl-dev python3-gdbm

2
setup.py
View File

@ -51,7 +51,7 @@ except:
setuptools.setup( setuptools.setup(
name='warcprox', name='warcprox',
version='2.0b2.dev26', version='2.0b2.dev27',
description='WARC writing MITM HTTP/S proxy', description='WARC writing MITM HTTP/S proxy',
url='https://github.com/internetarchive/warcprox', url='https://github.com/internetarchive/warcprox',
author='Noah Levitt', author='Noah Levitt',

4
warcprox/kafkafeed.py
View File

@ -32,7 +32,9 @@ class CaptureFeed:
def __init__(self, broker_list, topic=None): def __init__(self, broker_list, topic=None):
self.broker_list = broker_list self.broker_list = broker_list
self.topic = topic self.topic = topic
self._producer = kafka.KafkaProducer(bootstrap_servers=broker_list) # acks=0 to avoid ever blocking
self._producer = kafka.KafkaProducer(
bootstrap_servers=broker_list, acks=0)
def notify(self, recorded_url, records): def notify(self, recorded_url, records):
if records[0].type not in (b'revisit', b'response'): if records[0].type not in (b'revisit', b'response'):