From 95e92a112800bfe71d7b89e4cdf28075bde3542d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 8 Nov 2020 21:55:31 -0800 Subject: html: most small platform tweaks --- python/sandcrawler/html_ingest.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py index 958e81f..50b193c 100644 --- a/python/sandcrawler/html_ingest.py +++ b/python/sandcrawler/html_ingest.py @@ -206,6 +206,10 @@ def html_guess_platform(url: str, doc: HTMLParser, biblio: Optional[BiblioMetada try: if 'powered by PKP OJS' in doc.html: return "ojs" + if 'Powered by ' in doc.html: + return "arpha" + if "" in doc.html: + return "galenos" except UnicodeDecodeError: pass @@ -243,11 +247,6 @@ def html_guess_scope(url: str, doc: HTMLParser, biblio: Optional[BiblioMetadata] Unknown implies the page could be anything. "other" implies it is not fulltext or a landing page, but could be one of the other categories. - - TODO: known javascript-heavy single-page-app: - - https://riojournal.com/article/35913/ - - https://phmd.pl/resources/html/article/details?id=175497&language=en - - https://dez.pensoft.net/articles.php?id=11704 """ # basic paywall and loginwall detection based on URL -- cgit v1.2.3