From 7da130abec5ce9a8edbe6d2a4ef4c8be60e16a0d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 6 Mar 2019 11:47:21 -0800 Subject: fix harvester session.get() params --- python/fatcat_tools/harvest/doi_registrars.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'python/fatcat_tools/harvest') diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index 1aaad56a..802e0e22 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -87,15 +87,18 @@ class HarvestCrossrefWorker: date_str = date.isoformat() params = self.params(date_str) - headers = { - 'User-Agent': 'fatcat_tools/0.1.0 (https://fatcat.wiki; mailto:{}) python-requests'.format(self.contact_email), - } + http_session = requests_retry_session() + http_session.headers.update({ + 'User-Agent': 'fatcat_tools/0.1.0 (https://fatcat.wiki; mailto:{}) python-requests'.format( + self.contact_email), + }) count = 0 with produce_topic.get_producer() as producer: while True: - http_resp = requests_retry_session().get(self.api_host_url, params, headers=headers) + http_resp = http_session.get(self.api_host_url, params=params) if http_resp.status_code == 503: - # crude backoff + # crude backoff; now redundant with session exponential + # backoff, but allows for longer backoff/downtime on remote end print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code)) time.sleep(30.0) continue -- cgit v1.2.3