summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/harvest/oaipmh.py
diff options
context:
space:
mode:
Diffstat (limited to 'python/fatcat_tools/harvest/oaipmh.py')
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py16
1 files changed, 8 insertions, 8 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index ab424482..129df2e1 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -52,15 +52,15 @@ class HarvestOaiPmhWorker:
self.state = HarvestState(start_date, end_date)
self.state.initialize_from_kafka(self.state_topic, self.kafka_config)
- def kafka_produce_delivery_callback(err, msg):
- if err is not None:
- print("Kafka producer delivery error: {}".format(err))
- print("Bailing out...")
- # TODO: should it be sys.exit(-1)?
- raise KafkaException(err)
-
def fetch_date(self, date):
+ def fail_fast(err, msg):
+ if err is not None:
+ print("Kafka producer delivery error: {}".format(err))
+ print("Bailing out...")
+ # TODO: should it be sys.exit(-1)?
+ raise KafkaException(err)
+
producer = Producer(self.kafka_config)
api = sickle.Sickle(self.endpoint_url)
@@ -86,7 +86,7 @@ class HarvestOaiPmhWorker:
self.produce_topic,
item.raw.encode('utf-8'),
key=item.header.identifier.encode('utf-8'),
- on_delivery=self.kafka_produce_delivery_callback)
+ on_delivery=fail_fast)
producer.poll(0)
producer.flush()