diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2019-03-06 11:13:55 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-03-06 11:13:55 -0800 |
| commit | 7c93fa7ce5d60efe4f1694c1dedc051245017009 (patch) | |
| tree | 3103ada7a449a3473a6f9ab8eb315cd9688beca5 | |
| parent | d8f022d6d183607860b6d1013af0127de2b6c1ba (diff) | |
| download | fatcat-7c93fa7ce5d60efe4f1694c1dedc051245017009.tar.gz fatcat-7c93fa7ce5d60efe4f1694c1dedc051245017009.zip | |
10 MByte default Kafka produce (workers)
| -rw-r--r-- | python/fatcat_tools/workers/changelog.py | 8 | ||||
| -rw-r--r-- | python/fatcat_tools/workers/worker_common.py | 3 |
2 files changed, 9 insertions, 2 deletions
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index 39a84f18..6319d55a 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -33,7 +33,9 @@ class ChangelogWorker(FatcatWorker): else: self.offset = 1 - with topic.get_producer() as producer: + with topic.get_producer( + max_request_size=self.produce_max_request_size, + ) as producer: while True: latest = int(self.api.get_changelog(limit=1)[0].index) if latest > self.offset: @@ -85,7 +87,9 @@ class EntityUpdatesWorker(FatcatWorker): # using a sync producer to try and avoid racey loss of delivery (aka, # if consumer group updated but produce didn't stick) - with release_topic.get_sync_producer() as producer: + with release_topic.get_sync_producer( + max_request_size=self.produce_max_request_size, + ) as producer: for msg in consumer: cle = json.loads(msg.value.decode('utf-8')) #print(cle) diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py index 57fb710c..cb4e5dab 100644 --- a/python/fatcat_tools/workers/worker_common.py +++ b/python/fatcat_tools/workers/worker_common.py @@ -52,3 +52,6 @@ class FatcatWorker: self.kafka = KafkaClient(hosts=kafka_hosts, broker_version="1.0.0") self.produce_topic = produce_topic self.consume_topic = consume_topic + + # Kafka producer batch size tuning; also limit on size of single document + self.produce_max_request_size = 10000000 # 10 MByte-ish |