aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/html_metadata.py6
1 files changed, 4 insertions, 2 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index 2082b65..f9f48a6 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -483,7 +483,7 @@ def html_extract_biblio(doc_url: str, doc: HTMLParser) -> Optional[BiblioMetadat
for pattern in patterns:
val = head.css_first(pattern)
#print((field, pattern, val))
- if val and val.attrs.get('content'):
+ if val and 'content' in val.attrs and val.attrs['content']:
meta[field] = val.attrs['content']
break
@@ -492,7 +492,7 @@ def html_extract_biblio(doc_url: str, doc: HTMLParser) -> Optional[BiblioMetadat
val_list = head.css(pattern)
if val_list:
for val in val_list:
- if val.attrs.get('content'):
+ if 'content' in val.attrs and val.attrs['content']:
if not field in meta:
meta[field] = []
meta[field].append(val.attrs['content'])
@@ -587,6 +587,8 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name
for node in doc.css(selector):
for attr in attrs:
+ if not attr in node.attrs:
+ continue
url = node.attrs.get(attr)
if url:
resources.append(dict(url=url, type=type_name))