diff options
Diffstat (limited to 'grobid_tei_xml/parse.py')
| -rwxr-xr-x | grobid_tei_xml/parse.py | 204 |
1 files changed, 204 insertions, 0 deletions
diff --git a/grobid_tei_xml/parse.py b/grobid_tei_xml/parse.py new file mode 100755 index 0000000..a239e4d --- /dev/null +++ b/grobid_tei_xml/parse.py @@ -0,0 +1,204 @@ + +import io +import json +import xml.etree.ElementTree as ET +from typing import Any, AnyStr, Dict, List, Optional + +from .types import * + +xml_ns = "http://www.w3.org/XML/1998/namespace" +ns = "http://www.tei-c.org/ns/1.0" + + +def _string_to_tree(content: AnyStr) -> ET: + if isinstance(content, str): + return ET.parse(io.StringIO(content)) + elif isinstance(content, bytes): + return ET.parse(io.BytesIO(content)) + if isinstance(content, io.StringIO) or isinstance(content, io.BytesIO): + return ET.parse(content) + elif isinstance(content, ET): + return content + else: + raise TypeError(f"expected XML as string or bytes, got: {type(content)}") + +def _parse_authors(elem: Optional[ET.Element]) -> List[GrobidAffiliation]: + if not elem: + return [] + names = [] + for author in elem.findall(f".//{{{ns}}}author"): + pn = author.find(f"./{{{ns}}}persName") + if not pn: + continue + given_name = pn.findtext(f"./{{{ns}}}forename") or None + surname = pn.findtext(f"./{{{ns}}}surname") or None + full_name = " ".join([t.strip() for t in pn.itertext() + if t.strip()]).strip() + obj: Dict[str, Any] = dict(name=full_name) + if given_name: + obj["given_name"] = given_name + if surname: + obj["surname"] = surname + ae = author.find(f"./{{{ns}}}affiliation") + if ae: + affiliation: Dict[str, Any] = dict() + for on in ae.findall(f"./{{{ns}}}orgName"): + on_type = on.get("type") + if on_type: + affiliation[on_type] = on.text + addr_e = ae.find(f"./{{{ns}}}address") + if addr_e: + address = dict() + for t in addr_e.getchildren(): + address[t.tag.split("}")[-1]] = t.text + if address: + address['post_code'] = address.pop('postCode', None) + affiliation["address"] = GrobidAddress(**address) + # previously: + # affiliation['address'] = { + # 'post_code': addr.findtext('./{%s}postCode' % ns) or None, + # 'settlement': addr.findtext('./{%s}settlement' % ns) or None, + # 'country': addr.findtext('./{%s}country' % ns) or None, + # } + obj["affiliation"] = GrobidAffiliation(**affiliation) + names.append(GrobidAuthor(**obj)) + return names + +def _parse_citation(elem: ET.Element, ns: str = ns) -> GrobidCitation: + ref: Dict[str, Any] = dict() + ref["id"] = elem.attrib.get("{http://www.w3.org/XML/1998/namespace}id") + ref["unstructured"] = elem.findtext('.//{%s}note[@type="raw_reference"]' % + ns) + # Title stuff is messy in references... + ref["title"] = elem.findtext(f".//{{{ns}}}analytic/{{{ns}}}title") + other_title = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title") + if other_title: + if ref["title"]: + ref["journal"] = other_title + else: + ref["journal"] = None + ref["title"] = other_title + ref["authors"] = _parse_authors(elem) + ref["publisher"] = elem.findtext( + f".//{{{ns}}}publicationStmt/{{{ns}}}publisher") + if not ref["publisher"]: + ref["publisher"] = elem.findtext( + f".//{{{ns}}}imprint/{{{ns}}}publisher") + if ref["publisher"] == "": + ref["publisher"] = None + date = elem.find('.//{%s}date[@type="published"]' % ns) + ref["date"] = (date is not None) and date.attrib.get("when") + ref["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) + ref["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) + ref["doi"] = elem.findtext('.//{%s}idno[@type="DOI"]' % ns) + ref["arxiv_id"] = elem.findtext('.//{%s}idno[@type="arXiv"]' % ns) + if ref["arxiv_id"] and ref["arxiv_id"].startswith("arXiv:"): + ref["arxiv_id"] = ref["arxiv_id"][6:] + ref["pmcid"] = elem.findtext('.//{%s}idno[@type="PMCID"]' % ns) + ref["pmid"] = elem.findtext('.//{%s}idno[@type="PMID"]' % ns) + el = elem.find('.//{%s}biblScope[@unit="page"]' % ns) + if el is not None: + if el.attrib.get("from") and el.attrib.get("to"): + ref["pages"] = "{}-{}".format(el.attrib["from"], el.attrib["to"]) + else: + ref["pages"] = el.text + el = elem.find(".//{%s}ptr[@target]" % ns) + if el is not None: + ref["url"] = el.attrib["target"] + # Hand correction + # TODO: move this elsewhere + if ref["url"].endswith(".Lastaccessed"): + ref["url"] = ref["url"].replace(".Lastaccessed", "") + if ref["url"].startswith("<"): + ref["url"] = ref["url"][1:] + if ">" in ref["url"]: + ref["url"] = ref["url"].split(">")[0] + else: + ref["url"] = None + return GrobidCitation(**ref) + +def _parse_journal(elem: ET.Element, ns: str = ns) -> GrobidJournal: + journal = dict() + journal["name"] = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title") + journal["publisher"] = elem.findtext( + f".//{{{ns}}}publicationStmt/{{{ns}}}publisher") + if journal["publisher"] == "": + journal["publisher"] = None + journal["issn"] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns) + journal["eissn"] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns) + journal["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) + journal["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) + journal["abbrev"] = None + return GrobidJournal(**journal) + +def _parse_header(elem: ET.Element, ns: str = ns) -> GrobidHeader: + header = elem + info = dict() + info["title"] = header.findtext(f".//{{{ns}}}analytic/{{{ns}}}title") + info["authors"] = _parse_authors( + header.find(f".//{{{ns}}}sourceDesc/{{{ns}}}biblStruct")) + info["journal"] = _parse_journal(header) + date = header.find(f'.//{{{ns}}}date[@type="published"]') + info["date"] = (date is not None) and date.attrib.get("when") + info["doi"] = header.findtext(f'.//{{{ns}}}idno[@type="DOI"]') + if info["doi"]: + info["doi"] = info["doi"].lower() + return GrobidHeader(**info) + +def parse_document_xml(xml_text: AnyStr) -> GrobidDocument: + """ + Use this function to parse TEI-XML of a full document or header processed + by GROBID. + + Eg, the output of '/api/processFulltextDocument' or '/api/processHeader' + """ + tree = _string_to_tree(xml_text) + tei = tree.getroot() + info = dict() + encumbered = True + + header = tei.find(f".//{{{ns}}}teiHeader") + if header is None: + raise ValueError("XML does not look like TEI format") + + application_tag = header.findall( + f".//{{{ns}}}appInfo/{{{ns}}}application")[0] + + doc = GrobidDocument( + grobid_version=application_tag.attrib["version"].strip(), + grobid_timestamp=application_tag.attrib["when"].strip(), + header=_parse_header(header), + ) + + refs = [] + for (i, bs) in enumerate( + tei.findall(f".//{{{ns}}}listBibl/{{{ns}}}biblStruct")): + ref = _parse_citation(bs) + ref.index = i + refs.append(ref) + doc.citations = refs + + text = tei.find(f".//{{{ns}}}text") + # print(text.attrib) + if text and text.attrib.get(f"{{{xml_ns}}}lang"): + doc.language_code = text.attrib[f"{{{xml_ns}}}lang"] # xml:lang + + el = tei.find(f".//{{{ns}}}profileDesc/{{{ns}}}abstract") + doc.abstract = (el or None) and " ".join(el.itertext()).strip() + el = tei.find(f".//{{{ns}}}text/{{{ns}}}body") + doc.body = (el or None) and " ".join(el.itertext()).strip() + el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="acknowledgement"]') + doc.acknowledgement = (el or None) and " ".join( + el.itertext()).strip() + el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="annex"]') + doc.annex = (el or None) and " ".join(el.itertext()).strip() + + return doc + +def parse_citations_xml(xml_text: AnyStr) -> List[GrobidCitation]: + """ + Use this function to parse TEI-XML of one or more references. + + Eg, the output of '/api/processReferences' or '/api/processCitation'. + """ + tree = _string_to_tree(xml_text) |