From 5dc322c93eb3b92324c4f947697d2a2c69749040 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 6 Mar 2019 11:39:36 -0800 Subject: retry/backoff for Crossref harvester --- python/fatcat_tools/harvest/doi_registrars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/fatcat_tools/harvest/doi_registrars.py') diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index 55d85ef9..1aaad56a 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -10,7 +10,7 @@ import requests from pykafka import KafkaClient from fatcat_tools.workers import most_recent_message -from .harvest_common import HarvestState +from .harvest_common import HarvestState, requests_retry_session class HarvestCrossrefWorker: @@ -93,7 +93,7 @@ class HarvestCrossrefWorker: count = 0 with produce_topic.get_producer() as producer: while True: - http_resp = requests.get(self.api_host_url, params, headers=headers) + http_resp = requests_retry_session().get(self.api_host_url, params, headers=headers) if http_resp.status_code == 503: # crude backoff print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code)) -- cgit v1.2.3