aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-11-21 10:01:39 -0800
committerBryan Newbold <bnewbold@robocracy.org>2018-11-21 10:01:39 -0800
commitcf6e9789c850de101e1d1800b743333eab50b70e (patch)
treeecffdfee785fab3037db573dfa00e6ed31fbce7a
parent4527b82433b6278f9d090595ec172d98bceb752e (diff)
downloadfatcat-cf6e9789c850de101e1d1800b743333eab50b70e.tar.gz
fatcat-cf6e9789c850de101e1d1800b743333eab50b70e.zip
oaipmh: handle NoRecordsMatch
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py13
1 files changed, 8 insertions, 5 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index c3cb90db..f5e3fe7a 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -94,11 +94,14 @@ class HarvestOaiPmhWorker:
count = 0
with produce_topic.get_producer() as producer:
- for item in records:
- count += 1
- if count % 50 == 0:
- print("... up to {}".format(count))
- producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8'))
+ try:
+ for item in records:
+ count += 1
+ if count % 50 == 0:
+ print("... up to {}".format(count))
+ producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8'))
+ except sickle.oaiexceptions.NoRecordsMatch:
+ print("WARN: no OAI-PMH records for this date: {} (UTC)".format(date_str))
def run(self, continuous=False):