From 00ed69dd00d07344d62c5adad4e9d15c721c3bb1 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 8 Nov 2020 13:20:58 -0800 Subject: html: try to detect and mark XHTML (vs. HTML or XML) --- python/sandcrawler/ingest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/sandcrawler/ingest.py') diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index aedf2ff..c9b3d2f 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -554,7 +554,7 @@ class IngestFileWorker(SandcrawlerWorker): # here we split based on ingest type to try and extract a next hop html_ish_resource = bool( "html" in file_meta['mimetype'] - or "xhtml" in file_meta['mimetype'] + or "xhtml" in file_meta['mimetype'] # matches "application/xhtml+xml" or "application/xml" in file_meta['mimetype'] or "text/xml" in file_meta['mimetype'] ) @@ -653,7 +653,7 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = "wrong-mimetype" return result elif ingest_type == "html": - if file_meta['mimetype'] not in ("text/html",): + if file_meta['mimetype'] not in ("text/html", "application/xhtml+xml"): result['status'] = "wrong-mimetype" return result else: -- cgit v1.2.3