From 506fae13dbd111f89b4ae96c25cbd24ac1ec3ff0 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Tue, 14 Jan 2020 17:02:11 -0800
Subject: grobid-to-kafka support in ingest worker

---
 python/sandcrawler_worker.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/python/sandcrawler_worker.py b/python/sandcrawler_worker.py
index 26735ce..9199874 100755
--- a/python/sandcrawler_worker.py
+++ b/python/sandcrawler_worker.py
@@ -67,16 +67,22 @@ def run_persist_grobid(args):
 def run_ingest_file(args):
     consume_topic = "sandcrawler-{}.ingest-file-requests".format(args.env)
     produce_topic = "sandcrawler-{}.ingest-file-results".format(args.env)
+    grobid_topic = "sandcrawler-{}.grobid-output-pg".format(args.env)
     sink = KafkaSink(
         kafka_hosts=args.kafka_hosts,
         produce_topic=produce_topic,
     )
+    grobid_sink = KafkaSink(
+        kafka_hosts=args.kafka_hosts,
+        produce_topic=grobid_topic,
+    )
     grobid_client = GrobidClient(
         host_url=args.grobid_host,
     )
     worker = IngestFileWorker(
         grobid_client=grobid_client,
         sink=sink,
+        grobid_sink=grobid_sink,
     )
     pusher = KafkaJsonPusher(
         worker=worker,
-- 
cgit v1.2.3