aboutsummaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/harvest
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-04-08 21:10:45 -0700
committerBryan Newbold <bnewbold@robocracy.org>2019-09-20 11:21:10 -0700
commita8cde38d7f95908049d50d0b94ed33f3aa2cc75d (patch)
treed6fe260cb0de43c2d1f97f1cb82a136d4fb48f8a /python/fatcat_tools/harvest
parente0d7a847515314d5deb479074cc809b4aa46404c (diff)
downloadfatcat-a8cde38d7f95908049d50d0b94ed33f3aa2cc75d.tar.gz
fatcat-a8cde38d7f95908049d50d0b94ed33f3aa2cc75d.zip
small kafka tweaks for robustness
Diffstat (limited to 'python/fatcat_tools/harvest')
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py2
1 files changed, 2 insertions, 0 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index abf92792..133a3f01 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -113,6 +113,8 @@ class HarvestCrossrefWorker:
# crude backoff; now redundant with session exponential
# backoff, but allows for longer backoff/downtime on remote end
print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code))
+ # keep kafka producer connection alive
+ producer.poll(0)
time.sleep(30.0)
continue
http_resp.raise_for_status()