diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2019-03-06 11:13:55 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-03-06 11:13:55 -0800 |
| commit | 7c93fa7ce5d60efe4f1694c1dedc051245017009 (patch) | |
| tree | 3103ada7a449a3473a6f9ab8eb315cd9688beca5 /python/fatcat_tools/workers/worker_common.py | |
| parent | d8f022d6d183607860b6d1013af0127de2b6c1ba (diff) | |
| download | fatcat-7c93fa7ce5d60efe4f1694c1dedc051245017009.tar.gz fatcat-7c93fa7ce5d60efe4f1694c1dedc051245017009.zip | |
10 MByte default Kafka produce (workers)
Diffstat (limited to 'python/fatcat_tools/workers/worker_common.py')
| -rw-r--r-- | python/fatcat_tools/workers/worker_common.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py index 57fb710c..cb4e5dab 100644 --- a/python/fatcat_tools/workers/worker_common.py +++ b/python/fatcat_tools/workers/worker_common.py @@ -52,3 +52,6 @@ class FatcatWorker: self.kafka = KafkaClient(hosts=kafka_hosts, broker_version="1.0.0") self.produce_topic = produce_topic self.consume_topic = consume_topic + + # Kafka producer batch size tuning; also limit on size of single document + self.produce_max_request_size = 10000000 # 10 MByte-ish |