diff options
Diffstat (limited to 'python/sandcrawler/misc.py')
| -rw-r--r-- | python/sandcrawler/misc.py | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py index 1cdd038..38b2803 100644 --- a/python/sandcrawler/misc.py +++ b/python/sandcrawler/misc.py @@ -29,8 +29,10 @@ def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict: assert blob mimetype = magic.Magic(mime=True).from_buffer(blob) if mimetype in ("application/xml", "text/xml"): - # crude check for JATS XML, using only first 1 kB of file - if b"<article " in blob[:1024] and not b"<html" in blob[:1024]: + # crude checks for XHTML or JATS XML, using only first 1 kB of file + if b"<htm" in blob[:1024] and b'xmlns="http://www.w3.org/1999/xhtml"' in blob[:1024]: + mimetype = "application/xhtml+xml" + elif b"<article " in blob[:1024] and not b"<html" in blob[:1024]: mimetype = "application/jats+xml" hashes = [ hashlib.sha1(), |