aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/html_ingest.py2
-rw-r--r--python/sandcrawler/misc.py6
2 files changed, 5 insertions, 3 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py
index 6b9497b..03ec6f4 100644
--- a/python/sandcrawler/html_ingest.py
+++ b/python/sandcrawler/html_ingest.py
@@ -156,7 +156,7 @@ def fetch_html_resources(resources: List[dict], wayback_client: WaybackClient, w
wayback_resp = wayback_client.lookup_resource(resource['url'], closest=closest)
if not wayback_resp or wayback_resp.status != 'success':
raise NoCaptureError(f"HTML sub-resource not found: {resource['url']}")
- file_meta = gen_file_metadata(wayback_resp.body)
+ file_meta = gen_file_metadata(wayback_resp.body, allow_empty=True)
if file_meta['sha1hex'] != wayback_resp.cdx.sha1hex:
raise WaybackContentError("wayback payload sha1hex mismatch: {wayback_resp.cdx.url}")
full.append(WebResource(
diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py
index 67e5c0b..1cdd038 100644
--- a/python/sandcrawler/misc.py
+++ b/python/sandcrawler/misc.py
@@ -18,13 +18,15 @@ def clean_url(s: str) -> str:
parsed.colon_before_port = b''
return str(urlcanon.whatwg(parsed))
-def gen_file_metadata(blob: bytes) -> dict:
+def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict:
"""
Takes a file blob (bytestream) and returns hashes and other metadata.
Returns a dict: size_bytes, md5hex, sha1hex, sha256hex, mimetype
"""
- assert blob
+ assert blob is not None
+ if not allow_empty:
+ assert blob
mimetype = magic.Magic(mime=True).from_buffer(blob)
if mimetype in ("application/xml", "text/xml"):
# crude check for JATS XML, using only first 1 kB of file