aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2021-08-16 20:16:55 -0700
committerBryan Newbold <bnewbold@archive.org>2021-08-16 20:16:55 -0700
commit26d90505bda2d1dfcc25af6b8a0270faa11729e7 (patch)
tree9225b9206b34e6641be6438b6e16af99602309a7
parent8ccf0e6e935f169d3d6d35da36c767ebf0a4637a (diff)
downloadsandcrawler-26d90505bda2d1dfcc25af6b8a0270faa11729e7.tar.gz
sandcrawler-26d90505bda2d1dfcc25af6b8a0270faa11729e7.zip
html ingest: detect domain homepage (no path) as special case
-rw-r--r--python/sandcrawler/html_ingest.py8
1 files changed, 8 insertions, 0 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py
index 91b9cd6..115e7b5 100644
--- a/python/sandcrawler/html_ingest.py
+++ b/python/sandcrawler/html_ingest.py
@@ -236,6 +236,7 @@ def html_guess_scope(url: str, doc: HTMLParser, biblio: Optional[BiblioMetadata]
- component
- issue-fulltext
- landingpage
+ - homepage-domain
- blocked-paywall
- blocked-login
- blocked-captcha
@@ -249,6 +250,9 @@ def html_guess_scope(url: str, doc: HTMLParser, biblio: Optional[BiblioMetadata]
fulltext or a landing page, but could be one of the other categories.
"""
+ # assert that this is a real URL
+ assert url.count('/') >= 2
+
# basic paywall and loginwall detection based on URL
if url.endswith("/cookieAbsent"):
return "blocked-cookie"
@@ -264,6 +268,10 @@ def html_guess_scope(url: str, doc: HTMLParser, biblio: Optional[BiblioMetadata]
if "showcaptcha.asp" in url:
return "blocked-captcha"
+ # is this the top-level URL of the domain? aka, no path?
+ if url.count('/') <= 2 or (url.count('/') == 3) and url.endswith('/'):
+ return "homepage-domain"
+
platform = html_guess_platform(url, doc, biblio)
if biblio: