From 1ec661af75f37b3ae5031851f6c452039e08503c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 3 May 2022 17:35:52 -0700 Subject: ingest_tool: fix arg parsing --- python/ingest_tool.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/ingest_tool.py b/python/ingest_tool.py index ac9ece1..6b59611 100755 --- a/python/ingest_tool.py +++ b/python/ingest_tool.py @@ -70,11 +70,11 @@ def run_file_requests_backfill(args): Can be used to batch re-process known files. """ - grobid_topic = "sandcrawler-{}.grobid-output-pg".format(args.kafka_env) - pdftext_topic = "sandcrawler-{}.pdf-text".format(args.kafka_env) - thumbnail_topic = "sandcrawler-{}.pdf-thumbnail-180px-jpg".format(args.kafka_env) - xmldoc_topic = "sandcrawler-{}.xml-doc".format(args.kafka_env) - htmlteixml_topic = "sandcrawler-{}.html-teixml".format(args.kafka_env) + grobid_topic = "sandcrawler-{}.grobid-output-pg".format(args.env) + pdftext_topic = "sandcrawler-{}.pdf-text".format(args.env) + thumbnail_topic = "sandcrawler-{}.pdf-thumbnail-180px-jpg".format(args.env) + xmldoc_topic = "sandcrawler-{}.xml-doc".format(args.env) + htmlteixml_topic = "sandcrawler-{}.html-teixml".format(args.env) grobid_sink = KafkaSink( kafka_hosts=args.kafka_hosts, produce_topic=grobid_topic, @@ -143,6 +143,9 @@ def main(): action="store_true", help="report exceptions to Sentry", ) + parser.add_argument( + "--env", default="dev", help="environment (eg, prod, qa, dev)" + ) subparsers = parser.add_subparsers() sub_single = subparsers.add_parser("single", help="ingests a single base URL") @@ -211,9 +214,6 @@ def main(): default="localhost:9092", help="list of Kafka brokers (host/port) to use", ) - sub_file_requests_backfill.add_argument( - "--kafka-env", default="dev", help="Kafka topic namespace to use (eg, prod, qa, dev)" - ) sub_file_requests_backfill.add_argument( "--grobid-host", default="https://grobid.qa.fatcat.wiki", help="GROBID API host/port" ) -- cgit v1.2.3