diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-11-08 13:20:58 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-11-08 13:20:58 -0800 |
commit | 00ed69dd00d07344d62c5adad4e9d15c721c3bb1 (patch) | |
tree | 27f0298d6edd76a99686049470faeed1f54a0c69 | |
parent | 3977afdff906367525cd6959221b6f3edf19793d (diff) | |
download | sandcrawler-00ed69dd00d07344d62c5adad4e9d15c721c3bb1.tar.gz sandcrawler-00ed69dd00d07344d62c5adad4e9d15c721c3bb1.zip |
html: try to detect and mark XHTML (vs. HTML or XML)
-rw-r--r-- | python/sandcrawler/ingest.py | 4 | ||||
-rw-r--r-- | python/sandcrawler/misc.py | 6 |
2 files changed, 6 insertions, 4 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index aedf2ff..c9b3d2f 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -554,7 +554,7 @@ class IngestFileWorker(SandcrawlerWorker): # here we split based on ingest type to try and extract a next hop html_ish_resource = bool( "html" in file_meta['mimetype'] - or "xhtml" in file_meta['mimetype'] + or "xhtml" in file_meta['mimetype'] # matches "application/xhtml+xml" or "application/xml" in file_meta['mimetype'] or "text/xml" in file_meta['mimetype'] ) @@ -653,7 +653,7 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = "wrong-mimetype" return result elif ingest_type == "html": - if file_meta['mimetype'] not in ("text/html",): + if file_meta['mimetype'] not in ("text/html", "application/xhtml+xml"): result['status'] = "wrong-mimetype" return result else: diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py index 1cdd038..38b2803 100644 --- a/python/sandcrawler/misc.py +++ b/python/sandcrawler/misc.py @@ -29,8 +29,10 @@ def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict: assert blob mimetype = magic.Magic(mime=True).from_buffer(blob) if mimetype in ("application/xml", "text/xml"): - # crude check for JATS XML, using only first 1 kB of file - if b"<article " in blob[:1024] and not b"<html" in blob[:1024]: + # crude checks for XHTML or JATS XML, using only first 1 kB of file + if b"<htm" in blob[:1024] and b'xmlns="http://www.w3.org/1999/xhtml"' in blob[:1024]: + mimetype = "application/xhtml+xml" + elif b"<article " in blob[:1024] and not b"<html" in blob[:1024]: mimetype = "application/jats+xml" hashes = [ hashlib.sha1(), |