diff options
author | Martin Czygan <martin@archive.org> | 2020-07-10 21:32:41 +0000 |
---|---|---|
committer | Martin Czygan <martin@archive.org> | 2020-07-10 21:32:41 +0000 |
commit | 3c266e07771271241aa8cff3e3199a45109362af (patch) | |
tree | 73fa6aedf1bbfeffeac9c94593f5f9c4f2dd645b /python/fatcat_tools/harvest/oaipmh.py | |
parent | fdf1028c19b0623e30b91e49ffa65ed130dcfdc1 (diff) | |
parent | c9d8550be4bab808c2bad0b0d3642a71075202c0 (diff) | |
download | fatcat-3c266e07771271241aa8cff3e3199a45109362af.tar.gz fatcat-3c266e07771271241aa8cff3e3199a45109362af.zip |
datacite: resolve formatting issues in tests
Diffstat (limited to 'python/fatcat_tools/harvest/oaipmh.py')
-rw-r--r-- | python/fatcat_tools/harvest/oaipmh.py | 10 |
1 files changed, 1 insertions, 9 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index d30f9507..c4e4a82a 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -1,16 +1,9 @@ -import re import sys -import csv -import json import time -import itertools -import datetime -import requests import sickle from confluent_kafka import Producer, KafkaException -from fatcat_tools.workers import most_recent_message from .harvest_common import HarvestState @@ -31,7 +24,6 @@ class HarvestOaiPmhWorker: would want something similar operationally. Oh well! """ - def __init__(self, kafka_hosts, produce_topic, state_topic, start_date=None, end_date=None): @@ -69,7 +61,7 @@ class HarvestOaiPmhWorker: }) producer = Producer(producer_conf) - api = sickle.Sickle(self.endpoint_url) + api = sickle.Sickle(self.endpoint_url, max_retries=5, retry_status_codes=[503]) date_str = date.isoformat() # this dict kwargs hack is to work around 'from' as a reserved python keyword # recommended by sickle docs |