aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-11-20 00:05:04 -0800
committerBryan Newbold <bnewbold@archive.org>2020-11-20 00:05:04 -0800
commit07486ba6c7aad6934e8fe18f1565bb3f750276dd (patch)
tree49845542fed327e56c8632e6848e9591abfe7fa8 /python
parentc5436c73e2b75a9a936d53575caf66de26ef9195 (diff)
downloadsandcrawler-07486ba6c7aad6934e8fe18f1565bb3f750276dd.tar.gz
sandcrawler-07486ba6c7aad6934e8fe18f1565bb3f750276dd.zip
html: more conservative parsing of element attr
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/html_metadata.py6
1 files changed, 4 insertions, 2 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index 2082b65..f9f48a6 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -483,7 +483,7 @@ def html_extract_biblio(doc_url: str, doc: HTMLParser) -> Optional[BiblioMetadat
for pattern in patterns:
val = head.css_first(pattern)
#print((field, pattern, val))
- if val and val.attrs.get('content'):
+ if val and 'content' in val.attrs and val.attrs['content']:
meta[field] = val.attrs['content']
break
@@ -492,7 +492,7 @@ def html_extract_biblio(doc_url: str, doc: HTMLParser) -> Optional[BiblioMetadat
val_list = head.css(pattern)
if val_list:
for val in val_list:
- if val.attrs.get('content'):
+ if 'content' in val.attrs and val.attrs['content']:
if not field in meta:
meta[field] = []
meta[field].append(val.attrs['content'])
@@ -587,6 +587,8 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name
for node in doc.css(selector):
for attr in attrs:
+ if not attr in node.attrs:
+ continue
url = node.attrs.get(attr)
if url:
resources.append(dict(url=url, type=type_name))