From 80dc9bab9c6e40cdde95f9e9c7fad13ca64b0769 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 19 Sep 2019 17:11:47 -0700 Subject: small fixes to confluent-kafka importers/workers - decrease default changelog pipeline to 5.0sec - fix missing KafkaException harvester imports - more confluent-kafka tweaks - updates to kafka consumer configs - bump elastic updates consumergroup (again) --- python/fatcat_tools/workers/worker_common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'python/fatcat_tools/workers/worker_common.py') diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py index fb8cfc19..ef79f528 100644 --- a/python/fatcat_tools/workers/worker_common.py +++ b/python/fatcat_tools/workers/worker_common.py @@ -66,11 +66,7 @@ class FatcatWorker: self.api = api self.kafka_config = { 'bootstrap.servers': kafka_hosts, - 'delivery.report.only.error': True, - 'message.max.bytes': 20000000, # ~20 MBytes; broker is ~50 MBytes - 'default.topic.config': { - 'request.required.acks': 'all', - }, + 'message.max.bytes': 20000000, # ~20 MBytes; broker-side max is ~50 MBytes } self.produce_topic = produce_topic self.consume_topic = consume_topic -- cgit v1.2.3