aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/ingest.py4
-rw-r--r--python/sandcrawler/misc.py6
2 files changed, 6 insertions, 4 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index aedf2ff..c9b3d2f 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -554,7 +554,7 @@ class IngestFileWorker(SandcrawlerWorker):
# here we split based on ingest type to try and extract a next hop
html_ish_resource = bool(
"html" in file_meta['mimetype']
- or "xhtml" in file_meta['mimetype']
+ or "xhtml" in file_meta['mimetype'] # matches "application/xhtml+xml"
or "application/xml" in file_meta['mimetype']
or "text/xml" in file_meta['mimetype']
)
@@ -653,7 +653,7 @@ class IngestFileWorker(SandcrawlerWorker):
result['status'] = "wrong-mimetype"
return result
elif ingest_type == "html":
- if file_meta['mimetype'] not in ("text/html",):
+ if file_meta['mimetype'] not in ("text/html", "application/xhtml+xml"):
result['status'] = "wrong-mimetype"
return result
else:
diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py
index 1cdd038..38b2803 100644
--- a/python/sandcrawler/misc.py
+++ b/python/sandcrawler/misc.py
@@ -29,8 +29,10 @@ def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict:
assert blob
mimetype = magic.Magic(mime=True).from_buffer(blob)
if mimetype in ("application/xml", "text/xml"):
- # crude check for JATS XML, using only first 1 kB of file
- if b"<article " in blob[:1024] and not b"<html" in blob[:1024]:
+ # crude checks for XHTML or JATS XML, using only first 1 kB of file
+ if b"<htm" in blob[:1024] and b'xmlns="http://www.w3.org/1999/xhtml"' in blob[:1024]:
+ mimetype = "application/xhtml+xml"
+ elif b"<article " in blob[:1024] and not b"<html" in blob[:1024]:
mimetype = "application/jats+xml"
hashes = [
hashlib.sha1(),