From 31f237a6150f22676e93902a3597461ea954dc2c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 13 Nov 2019 00:28:13 -0800 Subject: crude support for 'sandcrawler' kafka topics --- python/fatcat_tools/importers/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'python/fatcat_tools') diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 74595790..c0740f94 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -711,6 +711,7 @@ class KafkaJsonPusher(RecordPusher): kafka_env, topic_suffix, group, + kafka_namespace=kwargs.get('kafka_namespace', 'fatcat') ) self.poll_interval = kwargs.get('poll_interval', 5.0) self.consume_batch_size = kwargs.get('consume_batch_size', 100) @@ -758,8 +759,8 @@ class KafkaJsonPusher(RecordPusher): return counts -def make_kafka_consumer(hosts, env, topic_suffix, group): - topic_name = "fatcat-{}.{}".format(env, topic_suffix) +def make_kafka_consumer(hosts, env, topic_suffix, group, kafka_namespace="fatcat"): + topic_name = "{}-{}.{}".format(kafka_namespace, env, topic_suffix) def fail_fast(err, partitions): if err is not None: -- cgit v1.2.3