differential wayback-error from wayback-content-error

The motivation here is to distinguish errors due to current content in
wayback (eg, in WARCs) from operational errors (eg, wayback machine is
down, or network failures/disruption).

1 files changed, 5 insertions, 1 deletions

diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 322859a..57988e8 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -8,7 +8,7 @@ import requests
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from collections import namedtuple
 
-from sandcrawler.ia import SavePageNowClient, CdxApiClient, WaybackClient, WaybackError, SavePageNowError, CdxApiError, PetaboxError, cdx_to_dict, ResourceResult
+from sandcrawler.ia import SavePageNowClient, CdxApiClient, WaybackClient, WaybackError, WaybackError, SavePageNowError, CdxApiError, PetaboxError, cdx_to_dict, ResourceResult
 from sandcrawler.grobid import GrobidClient
 from sandcrawler.pdfextract import process_pdf, PdfExtractResult
 from sandcrawler.misc import gen_file_metadata, clean_url
@@ -388,6 +388,10 @@ class IngestFileWorker(SandcrawlerWorker):
                 result['status'] = 'wayback-error'
                 result['error_message'] = str(e)[:1600]
                 return result
+            except WaybackContentError as e:
+                result['status'] = 'wayback-content-error'
+                result['error_message'] = str(e)[:1600]
+                return result
             except NotImplementedError as e:
                 result['status'] = 'not-implemented'
                 result['error_message'] = str(e)[:1600]