gen_file_metadata: allow empty/null bodies (if flag set)

This is for HTML sub-resources, which can validly be empty (I think)

1 files changed, 4 insertions, 2 deletions

diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py
index 67e5c0b..1cdd038 100644
--- a/python/sandcrawler/misc.py
+++ b/python/sandcrawler/misc.py
@@ -18,13 +18,15 @@ def clean_url(s: str) -> str:
         parsed.colon_before_port = b''
     return str(urlcanon.whatwg(parsed))
 
-def gen_file_metadata(blob: bytes) -> dict:
+def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict:
     """
     Takes a file blob (bytestream) and returns hashes and other metadata.
 
     Returns a dict: size_bytes, md5hex, sha1hex, sha256hex, mimetype
     """
-    assert blob
+    assert blob is not None
+    if not allow_empty:
+        assert blob
     mimetype = magic.Magic(mime=True).from_buffer(blob)
     if mimetype in ("application/xml", "text/xml"):
         # crude check for JATS XML, using only first 1 kB of file