diff options
| -rw-r--r-- | python/sandcrawler/workers.py | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index 632cb0a..f6693bb 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -178,7 +178,12 @@ class SandcrawlerFetchWorker(SandcrawlerWorker): blob = resp.content else: raise ValueError("not a CDX (wayback) or petabox (archive.org) dict; not sure how to proceed") - assert blob + if not blob: + return dict( + key=default_key, + source=record, + status="empty-blob", + ) return dict( key=default_key, status="success", |