aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2022-02-18 19:42:17 -0800
committerBryan Newbold <bnewbold@archive.org>2022-02-18 19:42:17 -0800
commit6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c (patch)
treec0bdb06e14d758c2d7fcff39fee5194d6ba955df
parent8993f5208a811b9f79013789d4e5150b7366421f (diff)
downloadsandcrawler-6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c.tar.gz
sandcrawler-6ff4dca52c413e66da10cac1fe3b5c7c3ee4315c.zip
ingest: handle more fileset failure modes
-rw-r--r--python/sandcrawler/fileset_strategies.py6
-rw-r--r--python/sandcrawler/ingest_fileset.py27
2 files changed, 30 insertions, 3 deletions
diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py
index 9bca551..cccc061 100644
--- a/python/sandcrawler/fileset_strategies.py
+++ b/python/sandcrawler/fileset_strategies.py
@@ -338,7 +338,11 @@ class WebFilesetStrategy(FilesetIngestStrategy):
continue
file_meta = gen_file_metadata(resource.body)
- file_meta, _html_resource = fix_transfer_encoding(file_meta, resource)
+ try:
+ file_meta, _html_resource = fix_transfer_encoding(file_meta, resource)
+ except:
+ m.status = "transfer-encoding-error"
+ continue
if self.ingest_strategy == "web-file":
file_file_meta = file_meta
diff --git a/python/sandcrawler/ingest_fileset.py b/python/sandcrawler/ingest_fileset.py
index 7337043..56f1697 100644
--- a/python/sandcrawler/ingest_fileset.py
+++ b/python/sandcrawler/ingest_fileset.py
@@ -256,7 +256,7 @@ class IngestFilesetWorker(IngestFileWorker):
result["status"] = "wrong-mimetype"
return result
else:
- # raise NotImplementedError()
+ # eg, datasets, components, etc
pass
result["_html_biblio"] = html_biblio
@@ -378,7 +378,30 @@ class IngestFilesetWorker(IngestFileWorker):
return result
# 3. Use strategy-specific methods to archive all files in platform manifest, and verify manifest metadata.
- archive_result = strategy_helper.process(dataset_meta)
+ try:
+ archive_result = strategy_helper.process(dataset_meta)
+ except SavePageNowError as e:
+ result["status"] = "spn2-error"
+ result["error_message"] = str(e)[:1600]
+ return result
+ except PetaboxError as e:
+ result["status"] = "petabox-error"
+ result["error_message"] = str(e)[:1600]
+ return result
+ except CdxApiError as e:
+ result["status"] = "cdx-error"
+ result["error_message"] = str(e)[:1600]
+ # add a sleep in cdx-error path as a slow-down
+ time.sleep(2.0)
+ return result
+ except WaybackError as e:
+ result["status"] = "wayback-error"
+ result["error_message"] = str(e)[:1600]
+ return result
+ except WaybackContentError as e:
+ result["status"] = "wayback-content-error"
+ result["error_message"] = str(e)[:1600]
+ return result
# 4. Summarize status and return structured result metadata.
result["status"] = archive_result.status