aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2021-08-24 14:08:10 -0400
committerBryan Newbold <bnewbold@archive.org>2021-08-24 14:08:10 -0400
commit6172700713c0ef19ef1da9f0c9d15d7ff29355a0 (patch)
treeb64d122118f7023f53da26616bc054471d40ed8f /python
parente1cde3c95e5176f232ecbc22a8619149078dc91f (diff)
downloadsandcrawler-6172700713c0ef19ef1da9f0c9d15d7ff29355a0.tar.gz
sandcrawler-6172700713c0ef19ef1da9f0c9d15d7ff29355a0.zip
HTML ingest: more meta-URI prefixes
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/html_metadata.py10
1 files changed, 8 insertions, 2 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index fb1e421..a1938e9 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -652,10 +652,16 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name
if not attr in node.attrs:
continue
url = node.attrs.get(attr)
- # special-case a couple meta urls which don't match with adblock rules
- if url in ['about:blank',]:
+ # special-case a couple meta URI prefixes which don't match with adblock rules
+ skip = False
+ for prefix in ['about:', 'data:', 'magnet:', 'urn:', 'mailto:']:
+ if url and url.startswith(prefix):
+ skip = True
+ break
+ if skip:
continue
if url:
+ print(url, file=sys.stderr)
resources.append(dict(url=url.strip(), type=type_name))
return resources