From 30905f1effb33c3ef193d084120aa3fbd75d0b9b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 1 Jul 2020 18:35:24 -0700 Subject: lint (flake8) tool python files --- python/fatcat_tools/harvest/oaipmh.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'python/fatcat_tools/harvest/oaipmh.py') diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index d30f9507..a7dc3d8c 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -1,16 +1,9 @@ -import re import sys -import csv -import json import time -import itertools -import datetime -import requests import sickle from confluent_kafka import Producer, KafkaException -from fatcat_tools.workers import most_recent_message from .harvest_common import HarvestState @@ -31,7 +24,6 @@ class HarvestOaiPmhWorker: would want something similar operationally. Oh well! """ - def __init__(self, kafka_hosts, produce_topic, state_topic, start_date=None, end_date=None): -- cgit v1.2.3 From c403cb4a1f20bd056008f68f71b374bde1e089b5 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Fri, 10 Jul 2020 00:54:55 +0200 Subject: arxiv: do retry five times of HTTP 503 --- python/fatcat_tools/harvest/oaipmh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/fatcat_tools/harvest/oaipmh.py') diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index a7dc3d8c..c4e4a82a 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -61,7 +61,7 @@ class HarvestOaiPmhWorker: }) producer = Producer(producer_conf) - api = sickle.Sickle(self.endpoint_url) + api = sickle.Sickle(self.endpoint_url, max_retries=5, retry_status_codes=[503]) date_str = date.isoformat() # this dict kwargs hack is to work around 'from' as a reserved python keyword # recommended by sickle docs -- cgit v1.2.3