small kafka tweaks for robustness

2 files changed, 5 insertions, 0 deletions

diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index abf92792..133a3f01 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -113,6 +113,8 @@ class HarvestCrossrefWorker:
                 # crude backoff; now redundant with session exponential
                 # backoff, but allows for longer backoff/downtime on remote end
                 print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code))
+                # keep kafka producer connection alive
+                producer.poll(0)
                 time.sleep(30.0)
                 continue
             http_resp.raise_for_status()
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 4648f6f3..0ca2617b 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -502,6 +502,9 @@ def make_kafka_consumer(hosts, env, topic_suffix, group):
     }
 
     def on_rebalance(consumer, partitions):
+        for p in partitions:
+            if p.error:
+                raise KafkaException(p.error)
         print("Kafka partitions rebalanced: {} / {}".format(
             consumer, partitions))