diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2021-08-24 14:08:10 -0400 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2021-08-24 14:08:10 -0400 |
| commit | 6172700713c0ef19ef1da9f0c9d15d7ff29355a0 (patch) | |
| tree | b64d122118f7023f53da26616bc054471d40ed8f /python | |
| parent | e1cde3c95e5176f232ecbc22a8619149078dc91f (diff) | |
| download | sandcrawler-6172700713c0ef19ef1da9f0c9d15d7ff29355a0.tar.gz sandcrawler-6172700713c0ef19ef1da9f0c9d15d7ff29355a0.zip | |
HTML ingest: more meta-URI prefixes
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/html_metadata.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py index fb1e421..a1938e9 100644 --- a/python/sandcrawler/html_metadata.py +++ b/python/sandcrawler/html_metadata.py @@ -652,10 +652,16 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name if not attr in node.attrs: continue url = node.attrs.get(attr) - # special-case a couple meta urls which don't match with adblock rules - if url in ['about:blank',]: + # special-case a couple meta URI prefixes which don't match with adblock rules + skip = False + for prefix in ['about:', 'data:', 'magnet:', 'urn:', 'mailto:']: + if url and url.startswith(prefix): + skip = True + break + if skip: continue if url: + print(url, file=sys.stderr) resources.append(dict(url=url.strip(), type=type_name)) return resources |