aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-11-21 10:06:26 -0800
committerBryan Newbold <bnewbold@robocracy.org>2018-11-21 10:06:26 -0800
commit5b0d7c1cd40a7a255886703c00d060eacfc27901 (patch)
tree802c37964cea980490473c8a26e713740d7f6862
parentcf6e9789c850de101e1d1800b743333eab50b70e (diff)
downloadfatcat-5b0d7c1cd40a7a255886703c00d060eacfc27901.tar.gz
fatcat-5b0d7c1cd40a7a255886703c00d060eacfc27901.zip
fix oai-pmh issue again
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py27
1 files changed, 14 insertions, 13 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index f5e3fe7a..4044ff10 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -86,22 +86,23 @@ class HarvestOaiPmhWorker:
produce_topic = self.kafka.topics[self.produce_topic]
# this dict kwargs hack is to work around 'from' as a reserved python keyword
# recommended by sickle docs
- records = api.ListRecords(**{
- 'metadataPrefix': self.metadata_prefix,
- 'from': date_str,
- 'until': date_str,
- })
+ try:
+ records = api.ListRecords(**{
+ 'metadataPrefix': self.metadata_prefix,
+ 'from': date_str,
+ 'until': date_str,
+ })
+ except sickle.oaiexceptions.NoRecordsMatch:
+ print("WARN: no OAI-PMH records for this date: {} (UTC)".format(date_str))
+ return
count = 0
with produce_topic.get_producer() as producer:
- try:
- for item in records:
- count += 1
- if count % 50 == 0:
- print("... up to {}".format(count))
- producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8'))
- except sickle.oaiexceptions.NoRecordsMatch:
- print("WARN: no OAI-PMH records for this date: {} (UTC)".format(date_str))
+ for item in records:
+ count += 1
+ if count % 50 == 0:
+ print("... up to {}".format(count))
+ producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8'))
def run(self, continuous=False):