grobid2json helper file

This file has been passed around a couple times and should probably be
published as a pypi.org project at some point.

1 files changed, 212 insertions, 0 deletions

diff --git a/fuzzycat/grobid2json.py b/fuzzycat/grobid2json.py
new file mode 100755
index 0000000..49f265a
--- /dev/null
+++ b/fuzzycat/grobid2json.py
@@ -0,0 +1,212 @@
+"""
+This file originally copied from the webgroup/sandcrawler repository.
+
+Parse GROBID TEI-XML and extract metadata into simple dict format. In
+particular, for fuzzycat, used with GROBID to parse raw ("unstructured")
+citation strings.
+"""
+
+import argparse
+import io
+import json
+import xml.etree.ElementTree as ET
+from typing import Any, AnyStr, Dict, List, Optional
+
+xml_ns = "http://www.w3.org/XML/1998/namespace"
+ns = "http://www.tei-c.org/ns/1.0"
+
+
+def all_authors(elem: Optional[ET.Element], ns: str = ns) -> List[Dict[str, Any]]:
+    if not elem:
+        return []
+    names = []
+    for author in elem.findall(".//{%s}author" % ns):
+        pn = author.find("./{%s}persName" % ns)
+        if not pn:
+            continue
+        given_name = pn.findtext("./{%s}forename" % ns) or None
+        surname = pn.findtext("./{%s}surname" % ns) or None
+        full_name = " ".join(pn.itertext())
+        obj: Dict[str, Any] = dict(name=full_name)
+        if given_name:
+            obj["given_name"] = given_name
+        if surname:
+            obj["surname"] = surname
+        ae = author.find("./{%s}affiliation" % ns)
+        if ae:
+            affiliation: Dict[str, Any] = dict()
+            for on in ae.findall("./{%s}orgName" % ns):
+                on_type = on.get("type")
+                if on_type:
+                    affiliation[on_type] = on.text
+            addr_e = ae.find("./{%s}address" % ns)
+            if addr_e:
+                address = dict()
+                for t in addr_e.getchildren():
+                    address[t.tag.split("}")[-1]] = t.text
+                if address:
+                    affiliation["address"] = address
+                # affiliation['address'] = {
+                #    'post_code': addr.findtext('./{%s}postCode' % ns) or None,
+                #    'settlement': addr.findtext('./{%s}settlement' % ns) or None,
+                #    'country': addr.findtext('./{%s}country' % ns) or None,
+                # }
+            obj["affiliation"] = affiliation
+        names.append(obj)
+    return names
+
+
+def journal_info(elem: ET.Element) -> Dict[str, Any]:
+    journal = dict()
+    journal["name"] = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title")
+    journal["publisher"] = elem.findtext(f".//{{{ns}}}publicationStmt/{{{ns}}}publisher")
+    if journal["publisher"] == "":
+        journal["publisher"] = None
+    journal["issn"] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns)
+    journal["eissn"] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns)
+    journal["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
+    journal["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
+    keys = list(journal.keys())
+
+    # remove empty/null keys
+    for k in keys:
+        if not journal[k]:
+            journal.pop(k)
+    return journal
+
+
+def biblio_info(elem: ET.Element, ns: str = ns) -> Dict[str, Any]:
+    ref: Dict[str, Any] = dict()
+    ref["id"] = elem.attrib.get("{http://www.w3.org/XML/1998/namespace}id")
+    ref["unstructured"] = elem.findtext('.//{%s}note[@type="raw_reference"]' % ns)
+    # Title stuff is messy in references...
+    ref["title"] = elem.findtext(f".//{{{ns}}}analytic/{{{ns}}}title")
+    other_title = elem.findtext(f".//{{{ns}}}monogr/{{{ns}}}title")
+    if other_title:
+        if ref["title"]:
+            ref["journal"] = other_title
+        else:
+            ref["journal"] = None
+            ref["title"] = other_title
+    ref["authors"] = all_authors(elem, ns=ns)
+    ref["publisher"] = elem.findtext(f".//{{{ns}}}publicationStmt/{{{ns}}}publisher")
+    if not ref["publisher"]:
+        ref["publisher"] = elem.findtext(f".//{{{ns}}}imprint/{{{ns}}}publisher")
+    if ref["publisher"] == "":
+        ref["publisher"] = None
+    date = elem.find('.//{%s}date[@type="published"]' % ns)
+    ref["date"] = (date is not None) and date.attrib.get("when")
+    ref["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
+    ref["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
+    ref["doi"] = elem.findtext('.//{%s}idno[@type="DOI"]' % ns)
+    ref["arxiv_id"] = elem.findtext('.//{%s}idno[@type="arXiv"]' % ns)
+    if ref["arxiv_id"] and ref["arxiv_id"].startswith("arXiv:"):
+        ref["arxiv_id"] = ref["arxiv_id"][6:]
+    ref["pmcid"] = elem.findtext('.//{%s}idno[@type="PMCID"]' % ns)
+    ref["pmid"] = elem.findtext('.//{%s}idno[@type="PMID"]' % ns)
+    el = elem.find('.//{%s}biblScope[@unit="page"]' % ns)
+    if el is not None:
+        if el.attrib.get("from") and el.attrib.get("to"):
+            ref["pages"] = "{}-{}".format(el.attrib["from"], el.attrib["to"])
+        else:
+            ref["pages"] = el.text
+    el = elem.find(".//{%s}ptr[@target]" % ns)
+    if el is not None:
+        ref["url"] = el.attrib["target"]
+        # Hand correction
+        if ref["url"].endswith(".Lastaccessed"):
+            ref["url"] = ref["url"].replace(".Lastaccessed", "")
+        if ref["url"].startswith("<"):
+            ref["url"] = ref["url"][1:]
+        if ">" in ref["url"]:
+            ref["url"] = ref["url"].split(">")[0]
+    else:
+        ref["url"] = None
+    return ref
+
+
+def teixml2json(content: AnyStr, encumbered: bool = True) -> Dict[str, Any]:
+
+    if isinstance(content, str):
+        tree = ET.parse(io.StringIO(content))
+    elif isinstance(content, bytes):
+        tree = ET.parse(io.BytesIO(content))
+
+    info: Dict[str, Any] = dict()
+
+    # print(content)
+    # print(content.getvalue())
+    tei = tree.getroot()
+
+    header = tei.find(".//{%s}teiHeader" % ns)
+    if header is None:
+        raise ValueError("XML does not look like TEI format")
+    application_tag = header.findall(f".//{{{ns}}}appInfo/{{{ns}}}application")[0]
+    info["grobid_version"] = application_tag.attrib["version"].strip()
+    info["grobid_timestamp"] = application_tag.attrib["when"].strip()
+    info["title"] = header.findtext(f".//{{{ns}}}analytic/{{{ns}}}title")
+    info["authors"] = all_authors(header.find(f".//{{{ns}}}sourceDesc/{{{ns}}}biblStruct"))
+    info["journal"] = journal_info(header)
+    date = header.find('.//{%s}date[@type="published"]' % ns)
+    info["date"] = (date is not None) and date.attrib.get("when")
+    info["fatcat_release"] = header.findtext('.//{%s}idno[@type="fatcat"]' % ns)
+    info["doi"] = header.findtext('.//{%s}idno[@type="DOI"]' % ns)
+    if info["doi"]:
+        info["doi"] = info["doi"].lower()
+
+    refs = []
+    for (i, bs) in enumerate(tei.findall(f".//{{{ns}}}listBibl/{{{ns}}}biblStruct")):
+        ref = biblio_info(bs)
+        ref["index"] = i
+        refs.append(ref)
+    info["citations"] = refs
+
+    text = tei.find(".//{%s}text" % (ns))
+    # print(text.attrib)
+    if text and text.attrib.get("{%s}lang" % xml_ns):
+        info["language_code"] = text.attrib["{%s}lang" % xml_ns]  # xml:lang
+
+    if encumbered:
+        el = tei.find(f".//{{{ns}}}profileDesc/{{{ns}}}abstract")
+        info["abstract"] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find(f".//{{{ns}}}text/{{{ns}}}body")
+        info["body"] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="acknowledgement"]')
+        info["acknowledgement"] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find(f'.//{{{ns}}}back/{{{ns}}}div[@type="annex"]')
+        info["annex"] = (el or None) and " ".join(el.itertext()).strip()
+
+    # remove empty/null keys
+    keys = list(info.keys())
+    for k in keys:
+        if not info[k]:
+            info.pop(k)
+    return info
+
+
+def main() -> None:  # pragma no cover
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="GROBID TEI XML to JSON",
+        usage="%(prog)s [options] <teifile>...",
+    )
+    parser.add_argument(
+        "--no-encumbered",
+        action="store_true",
+        help="don't include ambiguously copyright encumbered fields (eg, abstract, body)",
+    )
+    parser.add_argument("teifiles", nargs="+")
+
+    args = parser.parse_args()
+
+    for filename in args.teifiles:
+        content = open(filename, "r").read()
+        print(
+            json.dumps(
+                teixml2json(content, encumbered=(not args.no_encumbered)),
+                sort_keys=True,
+            ))
+
+
+if __name__ == "__main__":  # pragma no cover
+    main()