From a8cde38d7f95908049d50d0b94ed33f3aa2cc75d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 8 Apr 2019 21:10:45 -0700 Subject: small kafka tweaks for robustness --- python/fatcat_tools/harvest/doi_registrars.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'python/fatcat_tools/harvest/doi_registrars.py') diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index abf92792..133a3f01 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -113,6 +113,8 @@ class HarvestCrossrefWorker: # crude backoff; now redundant with session exponential # backoff, but allows for longer backoff/downtime on remote end print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code)) + # keep kafka producer connection alive + producer.poll(0) time.sleep(30.0) continue http_resp.raise_for_status() -- cgit v1.2.3