aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-11-08 11:00:11 -0800
committerBryan Newbold <bnewbold@archive.org>2020-11-08 11:00:13 -0800
commit19962108e0fd23fe6af24f170da8c47149e531dc (patch)
treeaa4faf7ed02ae1266e2d9d6b942ceba0e749ca7c
parentf74dbcd98c52cb5aa042c4f1a0daa7eddd42820b (diff)
downloadsandcrawler-19962108e0fd23fe6af24f170da8c47149e531dc.tar.gz
sandcrawler-19962108e0fd23fe6af24f170da8c47149e531dc.zip
ingest: fix null-body case
Broke this in earlier refactor.
-rw-r--r--python/sandcrawler/ia.py2
-rw-r--r--python/sandcrawler/ingest.py4
2 files changed, 6 insertions, 0 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 0b58f3b..da667b6 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -1076,6 +1076,8 @@ def fix_transfer_encoding(file_meta: dict, resource: ResourceResult) -> Tuple[di
if resource.body and file_meta['mimetype'] == 'application/gzip' and resource.cdx and resource.cdx.mimetype != 'application/gzip':
print(" transfer encoding not stripped: {}".format(resource.cdx.mimetype), file=sys.stderr)
inner_body = gzip.decompress(resource.body)
+ if not inner_body:
+ raise Exception("null body inside transfer encoding")
inner_resource = ResourceResult(
body=inner_body,
# copy all other fields
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 0c8eee6..2f9c523 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -531,6 +531,10 @@ class IngestFileWorker(SandcrawlerWorker):
result['status'] = 'blocked-cookie'
return result
+ if not resource.body:
+ result['status'] = 'null-body'
+ return result
+
file_meta = gen_file_metadata(resource.body)
try:
file_meta, resource = fix_transfer_encoding(file_meta, resource)