aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-04-08 21:10:45 -0700
committerBryan Newbold <bnewbold@robocracy.org>2019-04-08 21:10:45 -0700
commitaac6727d2fdf59e09c1f42da19a9a894304acc00 (patch)
tree153e97632af4a4d95b9b4defe0f5dc1884cb83bd /python
parentc6db48b47ac53d9ca05d2ce33f97fb73fb85131d (diff)
downloadfatcat-aac6727d2fdf59e09c1f42da19a9a894304acc00.tar.gz
fatcat-aac6727d2fdf59e09c1f42da19a9a894304acc00.zip
small kafka tweaks for robustness
Diffstat (limited to 'python')
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py2
-rw-r--r--python/fatcat_tools/importers/common.py3
2 files changed, 5 insertions, 0 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index abf92792..133a3f01 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -113,6 +113,8 @@ class HarvestCrossrefWorker:
# crude backoff; now redundant with session exponential
# backoff, but allows for longer backoff/downtime on remote end
print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code))
+ # keep kafka producer connection alive
+ producer.poll(0)
time.sleep(30.0)
continue
http_resp.raise_for_status()
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 4648f6f3..0ca2617b 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -502,6 +502,9 @@ def make_kafka_consumer(hosts, env, topic_suffix, group):
}
def on_rebalance(consumer, partitions):
+ for p in partitions:
+ if p.error:
+ raise KafkaException(p.error)
print("Kafka partitions rebalanced: {} / {}".format(
consumer, partitions))