diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2019-04-08 15:42:55 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-09-20 11:21:10 -0700 |
| commit | e0d7a847515314d5deb479074cc809b4aa46404c (patch) | |
| tree | 756dcd511262d7a5e3a454371bc75e967fc4c3cd /python/fatcat_import.py | |
| parent | 2e6cbe2d2ee444c465351b5af4f3f1455bc42bb5 (diff) | |
| download | fatcat-e0d7a847515314d5deb479074cc809b4aa46404c.tar.gz fatcat-e0d7a847515314d5deb479074cc809b4aa46404c.zip | |
convert importers to confluent-kafka library
Diffstat (limited to 'python/fatcat_import.py')
| -rwxr-xr-x | python/fatcat_import.py | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py index facf7eb9..7a0cd2ae 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -12,7 +12,8 @@ def run_crossref(args): edit_batch_size=args.batch_size, bezerk_mode=args.bezerk_mode) if args.kafka_mode: - KafkaJsonPusher(fci, args.kafka_hosts, args.kafka_env, "api-crossref", "fatcat-import").run() + KafkaJsonPusher(fci, args.kafka_hosts, args.kafka_env, "api-crossref", + "fatcat-import", edit_batch_size=args.batch_size).run() else: JsonLinePusher(fci, args.json_file).run() @@ -150,7 +151,7 @@ def main(): default="localhost:9092", help="list of Kafka brokers (host/port) to use") parser.add_argument('--kafka-env', - default="qa", + default="dev", help="Kafka topic namespace to use (eg, prod, qa)") parser.add_argument('--batch-size', help="size of batch to send", |