From ffdc901fa067db55fe6cfeb8d0c3807d29df092c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 15 Dec 2021 13:55:16 -0800 Subject: fileset ingest: more requests timeouts, sessions --- python/sandcrawler/ingest_fileset.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'python/sandcrawler/ingest_fileset.py') diff --git a/python/sandcrawler/ingest_fileset.py b/python/sandcrawler/ingest_fileset.py index 732a6ab..542dfbc 100644 --- a/python/sandcrawler/ingest_fileset.py +++ b/python/sandcrawler/ingest_fileset.py @@ -325,12 +325,18 @@ class IngestFilesetWorker(IngestFileWorker): result["error_message"] = str(e)[:1600] return result except requests.exceptions.HTTPError as e: + result["error_message"] = str(e)[:1600] if e.response.status_code == 404: result["status"] = "platform-404" result["error_message"] = str(e)[:1600] return result else: - raise e + result["status"] = "platform-http-error" + return result + except requests.exceptions.RequestException as e: + result["error_message"] = str(e)[:1600] + result["status"] = "platform-error" + return result # print(dataset_meta, file=sys.stderr) platform = dataset_meta.platform_name -- cgit v1.2.3