diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2018-11-21 10:01:39 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2018-11-21 10:01:39 -0800 |
| commit | cf6e9789c850de101e1d1800b743333eab50b70e (patch) | |
| tree | ecffdfee785fab3037db573dfa00e6ed31fbce7a | |
| parent | 4527b82433b6278f9d090595ec172d98bceb752e (diff) | |
| download | fatcat-cf6e9789c850de101e1d1800b743333eab50b70e.tar.gz fatcat-cf6e9789c850de101e1d1800b743333eab50b70e.zip | |
oaipmh: handle NoRecordsMatch
| -rw-r--r-- | python/fatcat_tools/harvest/oaipmh.py | 13 |
1 files changed, 8 insertions, 5 deletions
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index c3cb90db..f5e3fe7a 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -94,11 +94,14 @@ class HarvestOaiPmhWorker: count = 0 with produce_topic.get_producer() as producer: - for item in records: - count += 1 - if count % 50 == 0: - print("... up to {}".format(count)) - producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8')) + try: + for item in records: + count += 1 + if count % 50 == 0: + print("... up to {}".format(count)) + producer.produce(item.raw.encode('utf-8'), partition_key=item.header.identifier.encode('utf-8')) + except sickle.oaiexceptions.NoRecordsMatch: + print("WARN: no OAI-PMH records for this date: {} (UTC)".format(date_str)) def run(self, continuous=False): |