diff options
| -rw-r--r-- | python/sandcrawler/misc.py | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py index 8c91246..67e5c0b 100644 --- a/python/sandcrawler/misc.py +++ b/python/sandcrawler/misc.py @@ -26,6 +26,10 @@ def gen_file_metadata(blob: bytes) -> dict: """ assert blob mimetype = magic.Magic(mime=True).from_buffer(blob) + if mimetype in ("application/xml", "text/xml"): + # crude check for JATS XML, using only first 1 kB of file + if b"<article " in blob[:1024] and not b"<html" in blob[:1024]: + mimetype = "application/jats+xml" hashes = [ hashlib.sha1(), hashlib.sha256(), |