diff options
author | Bryan Newbold <bnewbold@archive.org> | 2022-02-18 19:42:17 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2022-02-18 19:42:17 -0800 |
commit | 6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c (patch) | |
tree | c0bdb06e14d758c2d7fcff39fee5194d6ba955df /python/sandcrawler/ingest_fileset.py | |
parent | 8993f5208a811b9f79013789d4e5150b7366421f (diff) | |
download | sandcrawler-6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c.tar.gz sandcrawler-6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c.zip |
ingest: handle more fileset failure modes
Diffstat (limited to 'python/sandcrawler/ingest_fileset.py')
-rw-r--r-- | python/sandcrawler/ingest_fileset.py | 27 |
1 files changed, 25 insertions, 2 deletions
diff --git a/python/sandcrawler/ingest_fileset.py b/python/sandcrawler/ingest_fileset.py index 7337043..56f1697 100644 --- a/python/sandcrawler/ingest_fileset.py +++ b/python/sandcrawler/ingest_fileset.py @@ -256,7 +256,7 @@ class IngestFilesetWorker(IngestFileWorker): result["status"] = "wrong-mimetype" return result else: - # raise NotImplementedError() + # eg, datasets, components, etc pass result["_html_biblio"] = html_biblio @@ -378,7 +378,30 @@ class IngestFilesetWorker(IngestFileWorker): return result # 3. Use strategy-specific methods to archive all files in platform manifest, and verify manifest metadata. - archive_result = strategy_helper.process(dataset_meta) + try: + archive_result = strategy_helper.process(dataset_meta) + except SavePageNowError as e: + result["status"] = "spn2-error" + result["error_message"] = str(e)[:1600] + return result + except PetaboxError as e: + result["status"] = "petabox-error" + result["error_message"] = str(e)[:1600] + return result + except CdxApiError as e: + result["status"] = "cdx-error" + result["error_message"] = str(e)[:1600] + # add a sleep in cdx-error path as a slow-down + time.sleep(2.0) + return result + except WaybackError as e: + result["status"] = "wayback-error" + result["error_message"] = str(e)[:1600] + return result + except WaybackContentError as e: + result["status"] = "wayback-content-error" + result["error_message"] = str(e)[:1600] + return result # 4. Summarize status and return structured result metadata. result["status"] = archive_result.status |