aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2021-08-16 20:15:47 -0700
committerBryan Newbold <bnewbold@archive.org>2021-08-16 20:15:49 -0700
commit8ccf0e6e935f169d3d6d35da36c767ebf0a4637a (patch)
tree22a050c9ab5388af213045ac981f79b82375281b /python
parenteab9b929a05da3fa25f4bfaffd84bb0d7b219c73 (diff)
downloadsandcrawler-8ccf0e6e935f169d3d6d35da36c767ebf0a4637a.tar.gz
sandcrawler-8ccf0e6e935f169d3d6d35da36c767ebf0a4637a.zip
html ingest: skip 'about:blank'
Couldn't get adblock rule matcher to match this, for some reason. maybe a special case?
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/html_metadata.py3
1 files changed, 3 insertions, 0 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index c805f0a..fb1e421 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -652,6 +652,9 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name
if not attr in node.attrs:
continue
url = node.attrs.get(attr)
+ # special-case a couple meta urls which don't match with adblock rules
+ if url in ['about:blank',]:
+ continue
if url:
resources.append(dict(url=url.strip(), type=type_name))