From 8e52c019bac85db1bcec743435fed4d346247959 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 10 Nov 2020 14:16:34 -0800 Subject: DOAJ and HTML ingest tweaks from QA run --- python/sandcrawler/html_ingest.py | 4 ++-- python/scripts/doaj2ingestrequest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py index 7594365..91b9cd6 100644 --- a/python/sandcrawler/html_ingest.py +++ b/python/sandcrawler/html_ingest.py @@ -25,9 +25,9 @@ def html_extract_body_teixml(doc: bytes) -> dict: include_comments=False, include_formatting=True, ) - except (ValueError, TypeError) as e: + except (ValueError, TypeError, Exception) as e: return dict( - status="parse-error", + status="trafilatura-parse-error", error_msg=str(e)[:1000], ) if tei_xml: diff --git a/python/scripts/doaj2ingestrequest.py b/python/scripts/doaj2ingestrequest.py index f1bae8c..b981ab6 100755 --- a/python/scripts/doaj2ingestrequest.py +++ b/python/scripts/doaj2ingestrequest.py @@ -74,7 +74,7 @@ def transform(obj: dict) -> List[dict]: doi: Optional[str] = None for ident in (bibjson['identifier'] or []): - if ident['type'].lower() == "doi" and ident['id'].startswith('10.'): + if ident['type'].lower() == "doi" and ident.get('id') and ident['id'].startswith('10.'): doi = ident['id'].lower() for link in (bibjson['link'] or []): -- cgit v1.2.3