From d18942d1ab4d394bdb275bcf9eb82d1cba814775 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Tue, 10 Mar 2020 12:55:47 +0100 Subject: common: use smaller batch size since XML parsing may be slow Address kafka tradeoff between long and short time-outs. Shorter time-outs would facilitate > consumer group re-balances and other consumer group state changes [...] in a reasonable human time-frame. --- python/fatcat_tools/importers/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 1cb5529e..5f5c46b8 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -735,7 +735,7 @@ class KafkaBs4XmlPusher(RecordPusher): kafka_namespace=kwargs.get('kafka_namespace', 'fatcat') ) self.poll_interval = kwargs.get('poll_interval', 5.0) - self.consume_batch_size = kwargs.get('consume_batch_size', 100) + self.consume_batch_size = kwargs.get('consume_batch_size', 25) def run(self): count = 0 -- cgit v1.2.3