diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2019-04-08 21:10:45 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-04-08 21:10:45 -0700 |
| commit | aac6727d2fdf59e09c1f42da19a9a894304acc00 (patch) | |
| tree | 153e97632af4a4d95b9b4defe0f5dc1884cb83bd | |
| parent | c6db48b47ac53d9ca05d2ce33f97fb73fb85131d (diff) | |
| download | fatcat-aac6727d2fdf59e09c1f42da19a9a894304acc00.tar.gz fatcat-aac6727d2fdf59e09c1f42da19a9a894304acc00.zip | |
small kafka tweaks for robustness
| -rw-r--r-- | python/fatcat_tools/harvest/doi_registrars.py | 2 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/common.py | 3 |
2 files changed, 5 insertions, 0 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index abf92792..133a3f01 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -113,6 +113,8 @@ class HarvestCrossrefWorker: # crude backoff; now redundant with session exponential # backoff, but allows for longer backoff/downtime on remote end print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code)) + # keep kafka producer connection alive + producer.poll(0) time.sleep(30.0) continue http_resp.raise_for_status() diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 4648f6f3..0ca2617b 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -502,6 +502,9 @@ def make_kafka_consumer(hosts, env, topic_suffix, group): } def on_rebalance(consumer, partitions): + for p in partitions: + if p.error: + raise KafkaException(p.error) print("Kafka partitions rebalanced: {} / {}".format( consumer, partitions)) |