html: try to detect and mark XHTML (vs. HTML or XML)

2 files changed, 6 insertions, 4 deletions

diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index aedf2ff..c9b3d2f 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -554,7 +554,7 @@ class IngestFileWorker(SandcrawlerWorker):
             # here we split based on ingest type to try and extract a next hop
             html_ish_resource = bool(
                 "html" in file_meta['mimetype']
-                or "xhtml" in file_meta['mimetype']
+                or "xhtml" in file_meta['mimetype'] # matches "application/xhtml+xml"
                 or "application/xml" in file_meta['mimetype']
                 or "text/xml" in file_meta['mimetype']
             )
@@ -653,7 +653,7 @@ class IngestFileWorker(SandcrawlerWorker):
                 result['status'] = "wrong-mimetype"
                 return result
         elif ingest_type == "html":
-            if file_meta['mimetype'] not in ("text/html",):
+            if file_meta['mimetype'] not in ("text/html", "application/xhtml+xml"):
                 result['status'] = "wrong-mimetype"
                 return result
         else:
diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py
index 1cdd038..38b2803 100644
--- a/python/sandcrawler/misc.py
+++ b/python/sandcrawler/misc.py
@@ -29,8 +29,10 @@ def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict:
         assert blob
     mimetype = magic.Magic(mime=True).from_buffer(blob)
     if mimetype in ("application/xml", "text/xml"):
-        # crude check for JATS XML, using only first 1 kB of file
-        if b"<article " in blob[:1024] and not b"<html" in blob[:1024]:
+        # crude checks for XHTML or JATS XML, using only first 1 kB of file
+        if b"<htm" in blob[:1024] and b'xmlns="http://www.w3.org/1999/xhtml"' in blob[:1024]:
+            mimetype = "application/xhtml+xml"
+        elif b"<article " in blob[:1024] and not b"<html" in blob[:1024]:
             mimetype = "application/jats+xml"
     hashes = [
         hashlib.sha1(),