diff options
| -rw-r--r-- | grobid_tei_xml/__init__.py | 1 | ||||
| -rw-r--r-- | grobid_tei_xml/grobid2json.py | 201 | ||||
| -rw-r--r-- | tests/test_grobid2json.py | 111 |
3 files changed, 0 insertions, 313 deletions
diff --git a/grobid_tei_xml/__init__.py b/grobid_tei_xml/__init__.py index 9ddce29..8cdbefc 100644 --- a/grobid_tei_xml/__init__.py +++ b/grobid_tei_xml/__init__.py @@ -1,5 +1,4 @@ __version__ = "0.1.0" -from .grobid2json import teixml2json from .parse import parse_citations_xml, parse_document_xml from .types import GrobidBiblio, GrobidDocument diff --git a/grobid_tei_xml/grobid2json.py b/grobid_tei_xml/grobid2json.py deleted file mode 100644 index c10de7c..0000000 --- a/grobid_tei_xml/grobid2json.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -NOTE: this file is DEPRECATED. It is only here for testing backwards -compatibility, and will be removed soon. -""" - -import io -import xml.etree.ElementTree as ET -from typing import Any, AnyStr, Dict, List, Optional - -xml_ns = "http://www.w3.org/XML/1998/namespace" -ns = "http://www.tei-c.org/ns/1.0" - - -def all_authors(elem: Optional[ET.Element], ns: str = ns) -> List[Dict[str, Any]]: - if not elem: - return [] - names = [] - for author in elem.findall(".//{%s}author" % ns): - pn = author.find("./{%s}persName" % ns) - if not pn: - continue - given_name = pn.findtext("./{%s}forename" % ns) or None - surname = pn.findtext("./{%s}surname" % ns) or None - full_name = " ".join([t.strip() for t in pn.itertext() if t.strip()]).strip() - obj: Dict[str, Any] = dict(name=full_name) - if given_name: - obj["given_name"] = given_name - if surname: - obj["surname"] = surname - ae = author.find("./{%s}affiliation" % ns) - if ae: - affiliation: Dict[str, Any] = dict() - for on in ae.findall("./{%s}orgName" % ns): - on_type = on.get("type") - if on_type: - affiliation[on_type] = on.text - addr_e = ae.find("./{%s}address" % ns) - if addr_e: - address = dict() - for t in list(addr_e): - address[t.tag.split("}")[-1]] = t.text - if address: - affiliation["address"] = address - # affiliation['address'] = { - # 'post_code': addr.findtext('./{%s}postCode' % ns) or None, - # 'settlement': addr.findtext('./{%s}settlement' % ns) or None, - # 'country': addr.findtext('./{%s}country' % ns) or None, - # } - obj["affiliation"] = affiliation - names.append(obj) - return names - - -def journal_info(elem: ET.Element) -> Dict[str, Any]: - journal = dict() - journal["name"] = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title") - journal["publisher"] = elem.findtext(f".//{{{ns}}}publicationStmt/{{{ns}}}publisher") - if journal["publisher"] == "": - journal["publisher"] = None - journal["issn"] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns) - journal["eissn"] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns) - journal["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) - journal["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) - - # remove empty/null keys - keys = list(journal.keys()) - for k in keys: - if not journal[k]: - journal.pop(k) - return journal - - -def biblio_info(elem: ET.Element, ns: str = ns) -> Dict[str, Any]: - ref: Dict[str, Any] = dict() - ref["id"] = elem.attrib.get("{http://www.w3.org/XML/1998/namespace}id") - ref["unstructured"] = elem.findtext('.//{%s}note[@type="raw_reference"]' % ns) - # Title stuff is messy in references... - ref["title"] = elem.findtext(f".//{{{ns}}}analytic/{{{ns}}}title") - other_title = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title") - if other_title: - if ref["title"]: - ref["journal"] = other_title - else: - ref["journal"] = None - ref["title"] = other_title - ref["authors"] = all_authors(elem, ns=ns) - ref["publisher"] = elem.findtext(f".//{{{ns}}}publicationStmt/{{{ns}}}publisher") - if not ref["publisher"]: - ref["publisher"] = elem.findtext(f".//{{{ns}}}imprint/{{{ns}}}publisher") - if ref["publisher"] == "": - ref["publisher"] = None - date = elem.find('.//{%s}date[@type="published"]' % ns) - ref["date"] = (date is not None) and date.attrib.get("when") - ref["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) - ref["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) - ref["doi"] = elem.findtext('.//{%s}idno[@type="DOI"]' % ns) - ref["arxiv_id"] = elem.findtext('.//{%s}idno[@type="arXiv"]' % ns) - if ref["arxiv_id"] and ref["arxiv_id"].startswith("arXiv:"): - ref["arxiv_id"] = ref["arxiv_id"][6:] - ref["pmcid"] = elem.findtext('.//{%s}idno[@type="PMCID"]' % ns) - ref["pmid"] = elem.findtext('.//{%s}idno[@type="PMID"]' % ns) - el = elem.find('.//{%s}biblScope[@unit="page"]' % ns) - if el is not None: - if el.attrib.get("from") and el.attrib.get("to"): - ref["pages"] = "{}-{}".format(el.attrib["from"], el.attrib["to"]) - else: - ref["pages"] = el.text - el = elem.find(".//{%s}ptr[@target]" % ns) - if el is not None: - ref["url"] = el.attrib["target"] - # Hand correction - # TODO: move this elsewhere - if ref["url"].endswith(".Lastaccessed"): - ref["url"] = ref["url"].replace(".Lastaccessed", "") - if ref["url"].startswith("<"): - ref["url"] = ref["url"][1:] - if ">" in ref["url"]: - ref["url"] = ref["url"].split(">")[0] - else: - ref["url"] = None - - # remove empty/null keys - keys = list(ref.keys()) - for k in keys: - if ref[k] is None: - ref.pop(k) - return ref - - -def teixml2json(content: AnyStr, encumbered: bool = True) -> Dict[str, Any]: - - if isinstance(content, str): - tree = ET.parse(io.StringIO(content)) - elif isinstance(content, bytes): - tree = ET.parse(io.BytesIO(content)) - - info: Dict[str, Any] = dict() - - # print(content) - # print(content.getvalue()) - tei = tree.getroot() - - header = tei.find(".//{%s}teiHeader" % ns) - if header is None: - raise ValueError("XML does not look like TEI format") - application_tag = header.findall(f".//{{{ns}}}appInfo/{{{ns}}}application")[0] - info["grobid_version"] = application_tag.attrib["version"].strip() - info["grobid_timestamp"] = application_tag.attrib["when"].strip() - info["title"] = header.findtext(f".//{{{ns}}}analytic/{{{ns}}}title") - info["authors"] = all_authors(header.find(f".//{{{ns}}}sourceDesc/{{{ns}}}biblStruct")) - info["journal"] = journal_info(header) - date = header.find('.//{%s}date[@type="published"]' % ns) - info["date"] = (date is not None) and date.attrib.get("when") - info["fatcat_release"] = header.findtext('.//{%s}idno[@type="fatcat"]' % ns) - info["doi"] = header.findtext('.//{%s}idno[@type="DOI"]' % ns) - if info["doi"]: - info["doi"] = info["doi"].lower() - - refs = [] - for (i, bs) in enumerate(tei.findall(f".//{{{ns}}}listBibl/{{{ns}}}biblStruct")): - ref = biblio_info(bs) - ref["index"] = i - refs.append(ref) - info["citations"] = refs - - text = tei.find(".//{%s}text" % (ns)) - # print(text.attrib) - if text and text.attrib.get("{%s}lang" % xml_ns): - info["language_code"] = text.attrib["{%s}lang" % xml_ns] # xml:lang - - if encumbered: - el = tei.find(f".//{{{ns}}}profileDesc/{{{ns}}}abstract") - info["abstract"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find(f".//{{{ns}}}text/{{{ns}}}body") - info["body"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="acknowledgement"]') - info["acknowledgement"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="annex"]') - info["annex"] = (el or None) and " ".join(el.itertext()).strip() - - # remove empty/null keys - keys = list(info.keys()) - for k in keys: - if not info[k]: - info.pop(k) - return info - - -def transform_grobid_ref_xml(raw_xml: str) -> Optional[dict]: - """ - Parses GROBID XML for the case of a single reference/citation string (eg, - not a full/propper TEI-XML fulltext document), and returns a dict. - """ - # first, remove any xmlns stuff, for consistent parsing - raw_xml = raw_xml.replace('xmlns="http://www.tei-c.org/ns/1.0"', "") - tree = ET.parse(io.StringIO(raw_xml)) - root = tree.getroot() - ref = biblio_info(root, ns="") - if not any(ref.values()): - return None - return ref diff --git a/tests/test_grobid2json.py b/tests/test_grobid2json.py deleted file mode 100644 index 47ab293..0000000 --- a/tests/test_grobid2json.py +++ /dev/null @@ -1,111 +0,0 @@ -import json -import xml - -import pytest - -from grobid_tei_xml.grobid2json import teixml2json, transform_grobid_ref_xml - - -def test_small_xml() -> None: - - with open('tests/files/small.xml', 'r') as f: - tei_xml = f.read() - with open('tests/files/small.json', 'r') as f: - json_form = json.loads(f.read()) - - assert teixml2json(tei_xml) == json_form - - -def test_invalid_xml() -> None: - - with pytest.raises(xml.etree.ElementTree.ParseError): - teixml2json("this is not XML") - with pytest.raises(ValueError): - teixml2json("<xml></xml>") - - -def test_grobid_teixml2json() -> None: - - with open("tests/files/example_grobid.tei.xml", "r") as f: - blob = f.read() - - obj = teixml2json(blob, True) - - assert obj[ - "title"] == """Changes of patients' satisfaction with the health care services in Lithuanian Health Promoting Hospitals network""" - - ref = [c for c in obj["citations"] if c["id"] == "b12"][0] - assert ref["authors"][0] == {"given_name": "K", "name": "K Tasa", "surname": "Tasa"} - assert ref["journal"] == "Quality Management in Health Care" - assert ref["title"] == "Using patient feedback for quality improvement" - assert ref["date"] == "1996" - assert ref["pages"] == "206-225" - assert ref["volume"] == "8" - assert ref["unstructured"] == \ - """Tasa K, Baker R, Murray M. Using patient feedback for qua- lity improvement. Quality Management in Health Care 1996;8:206-19.""" - - -def test_transform_grobid_ref_xml() -> None: - citation_xml = """ -<biblStruct > - <analytic> - <title level="a" type="main">Mesh migration following abdominal hernia repair: a comprehensive review</title> - <author> - <persName - xmlns="http://www.tei-c.org/ns/1.0"> - <forename type="first">H</forename> - <forename type="middle">B</forename> - <surname>Cunningham</surname> - </persName> - </author> - <author> - <persName - xmlns="http://www.tei-c.org/ns/1.0"> - <forename type="first">J</forename> - <forename type="middle">J</forename> - <surname>Weis</surname> - </persName> - </author> - <author> - <persName - xmlns="http://www.tei-c.org/ns/1.0"> - <forename type="first">L</forename> - <forename type="middle">R</forename> - <surname>Taveras</surname> - </persName> - </author> - <author> - <persName - xmlns="http://www.tei-c.org/ns/1.0"> - <forename type="first">S</forename> - <surname>Huerta</surname> - </persName> - </author> - <idno type="DOI">10.1007/s10029-019-01898-9</idno> - <idno type="PMID">30701369</idno> - </analytic> - <monogr> - <title level="j">Hernia</title> - <imprint> - <biblScope unit="volume">23</biblScope> - <biblScope unit="issue">2</biblScope> - <biblScope unit="page" from="235" to="243" /> - <date type="published" when="2019-01-30" /> - </imprint> - </monogr> -</biblStruct>""" - - d = transform_grobid_ref_xml(citation_xml) - assert d - assert d['title'] == \ - "Mesh migration following abdominal hernia repair: a comprehensive review" - assert d['authors'][2]['given_name'] == "L" - assert d['authors'][2]['surname'] == "Taveras" - assert d['authors'][2]['name'] == "L R Taveras" - assert d['doi'] == "10.1007/s10029-019-01898-9" - assert d['pmid'] == "30701369" - assert d['date'] == "2019-01-30" - assert d['pages'] == "235-243" - assert d['volume'] == "23" - assert d['issue'] == "2" - assert d['journal'] == "Hernia" |