From a3719d6501e4b7cc520ba05264d99bdcc2d7bed7 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 15 Dec 2021 16:29:47 -0800 Subject: more fileset ingest tweaks --- python/sandcrawler/fileset_platforms.py | 2 ++ python/sandcrawler/fileset_strategies.py | 5 +++++ 2 files changed, 7 insertions(+) (limited to 'python') diff --git a/python/sandcrawler/fileset_platforms.py b/python/sandcrawler/fileset_platforms.py index 81aad87..4adc9b7 100644 --- a/python/sandcrawler/fileset_platforms.py +++ b/python/sandcrawler/fileset_platforms.py @@ -205,6 +205,8 @@ class DataverseHelper(FilesetPlatformHelper): ) resp.raise_for_status() obj = resp.json() + if not "latestVersion" in obj["data"]: + raise PlatformScopeError("could not find latest version for dataverse record") obj_latest = obj["data"]["latestVersion"] dataset_version = ( f"{obj_latest['versionNumber']}.{obj_latest['versionMinorNumber']}" diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index f83d1ce..9696f3c 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -150,6 +150,11 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy): m.status = "downloaded-local" except requests.exceptions.RequestException: m.status = "error-platform-download" + return ArchiveStrategyResult( + ingest_strategy=self.ingest_strategy, + manifest=item.manifest, + status="error-platform-download", + ) print(f" verifying {m.path}", file=sys.stderr) file_meta = gen_file_metadata_path(local_path, allow_empty=True) -- cgit v1.2.3