From 55bd186a21f5e3703e8f3ba3b0a14f1387ed0ccc Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 15 Feb 2021 21:51:06 -0800 Subject: increase indexing timeout to 50sec (60sec for kafka) --- fatcat_scholar/kafka.py | 2 +- fatcat_scholar/worker.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fatcat_scholar/kafka.py b/fatcat_scholar/kafka.py index 5faff9a..e71bc3d 100644 --- a/fatcat_scholar/kafka.py +++ b/fatcat_scholar/kafka.py @@ -60,7 +60,7 @@ class KafkaWorker(object): self.counts: Counter = Counter() self.kafka_brokers = kafka_brokers self.batch_size = kwargs.get("batch_size", 1) - self.batch_timeout_sec = kwargs.get("batch_timeout_sec", 30) + self.batch_timeout_sec = kwargs.get("batch_timeout_sec", 60) self.poll_interval_sec = kwargs.get("poll_interval_sec", 5.0) self.consumer = self.create_kafka_consumer( kafka_brokers, consume_topics, consumer_group diff --git a/fatcat_scholar/worker.py b/fatcat_scholar/worker.py index 854c1a2..823f1bd 100644 --- a/fatcat_scholar/worker.py +++ b/fatcat_scholar/worker.py @@ -153,7 +153,7 @@ class IndexDocsWorker(KafkaWorker): if not bulk_actions: return - elasticsearch.helpers.bulk(self.es_client, bulk_actions, timeout="30s") + elasticsearch.helpers.bulk(self.es_client, bulk_actions, timeout="50s") self.counts["batches-indexed"] += 1 @@ -236,7 +236,7 @@ def main() -> None: fdw.run() elif args.worker == "index-docs-worker": es_client = elasticsearch.Elasticsearch( - settings.ELASTICSEARCH_WRITE_BASE, timeout=25.0 + settings.ELASTICSEARCH_WRITE_BASE, timeout=50.0 ) idw = IndexDocsWorker( kafka_brokers=settings.KAFKA_BROKERS, -- cgit v1.2.3