summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/harvest/oaipmh.py
diff options
context:
space:
mode:
authorMartin Czygan <martin@archive.org>2020-07-10 21:32:41 +0000
committerMartin Czygan <martin@archive.org>2020-07-10 21:32:41 +0000
commit3c266e07771271241aa8cff3e3199a45109362af (patch)
tree73fa6aedf1bbfeffeac9c94593f5f9c4f2dd645b /python/fatcat_tools/harvest/oaipmh.py
parentfdf1028c19b0623e30b91e49ffa65ed130dcfdc1 (diff)
parentc9d8550be4bab808c2bad0b0d3642a71075202c0 (diff)
downloadfatcat-3c266e07771271241aa8cff3e3199a45109362af.tar.gz
fatcat-3c266e07771271241aa8cff3e3199a45109362af.zip
datacite: resolve formatting issues in tests
Diffstat (limited to 'python/fatcat_tools/harvest/oaipmh.py')
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py10
1 files changed, 1 insertions, 9 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index d30f9507..c4e4a82a 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -1,16 +1,9 @@
-import re
import sys
-import csv
-import json
import time
-import itertools
-import datetime
-import requests
import sickle
from confluent_kafka import Producer, KafkaException
-from fatcat_tools.workers import most_recent_message
from .harvest_common import HarvestState
@@ -31,7 +24,6 @@ class HarvestOaiPmhWorker:
would want something similar operationally. Oh well!
"""
-
def __init__(self, kafka_hosts, produce_topic, state_topic,
start_date=None, end_date=None):
@@ -69,7 +61,7 @@ class HarvestOaiPmhWorker:
})
producer = Producer(producer_conf)
- api = sickle.Sickle(self.endpoint_url)
+ api = sickle.Sickle(self.endpoint_url, max_retries=5, retry_status_codes=[503])
date_str = date.isoformat()
# this dict kwargs hack is to work around 'from' as a reserved python keyword
# recommended by sickle docs