diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-11-14 12:20:16 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-11-14 12:20:19 -0800 |
| commit | 76b504e3c8d42e34509d75be4a4df69ae9f15a09 (patch) | |
| tree | 40c18fdfb148e8461a9ed9557c9640a6e1aa800c /python | |
| parent | 87f9759dd3e5e6f4e7663cf8be685fc3b8ad3400 (diff) | |
| download | sandcrawler-76b504e3c8d42e34509d75be4a4df69ae9f15a09.tar.gz sandcrawler-76b504e3c8d42e34509d75be4a4df69ae9f15a09.zip | |
bump kafka max poll interval for consumers
The ingest worker keeps timing out at just over 5 minutes, so bump it
just a bit.
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/workers.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index def890e..a23d9a4 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -406,8 +406,8 @@ def make_kafka_consumer(hosts, consume_topic, group): 'enable.auto.offset.store': False, 'enable.auto.commit': True, # user code timeout; if no poll after this long, assume user code - # hung and rebalance (default: 5min) - 'max.poll.interval.ms': 300000, + # hung and rebalance (default: 6min) + 'max.poll.interval.ms': 360000, 'default.topic.config': { 'auto.offset.reset': 'latest', }, |