diff options
| -rw-r--r-- | python/sandcrawler/ingest.py | 13 |
1 files changed, 12 insertions, 1 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 11b8a4c..bcb6608 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -70,7 +70,7 @@ class IngestFileWorker(SandcrawlerWorker): """ if not self.try_existing_ingest: return None - raise NotImplementedError + raise NotImplementedError("can't pre-check ingests yet") # this "return True" is just here to make pylint happy return True @@ -82,6 +82,13 @@ class IngestFileWorker(SandcrawlerWorker): """ via = "none" resource = None + + if url.startswith("http://web.archive.org/web/") or url.startswith("https://web.archive.org/web/"): + raise NotImplementedError("handling direct wayback links not supported yet") + + if url.startswith("http://archive.org/") or url.startswith("https://archive.org/"): + raise NotImplementedError("fetching from archive.org not implemented yet") + if self.try_wayback: via = "wayback" resource = self.wayback_client.lookup_resource(url, best_mimetype) @@ -192,6 +199,10 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = 'wayback-error' result['error_message'] = str(e)[:1600] return result + except NotImplementedError as e: + result['status'] = 'not-implemented' + result['error_message'] = str(e)[:1600] + return result if not resource.hit: result['status'] = resource.status |