From 8ccf0e6e935f169d3d6d35da36c767ebf0a4637a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 16 Aug 2021 20:15:47 -0700 Subject: html ingest: skip 'about:blank' Couldn't get adblock rule matcher to match this, for some reason. maybe a special case? --- python/sandcrawler/html_metadata.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'python/sandcrawler') diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py index c805f0a..fb1e421 100644 --- a/python/sandcrawler/html_metadata.py +++ b/python/sandcrawler/html_metadata.py @@ -652,6 +652,9 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name if not attr in node.attrs: continue url = node.attrs.get(attr) + # special-case a couple meta urls which don't match with adblock rules + if url in ['about:blank',]: + continue if url: resources.append(dict(url=url.strip(), type=type_name)) -- cgit v1.2.3