remove grobid2json helper file, replace with grobid_tei_xml

7 files changed, 22 insertions, 224 deletions

diff --git a/python/.coveragerc b/python/.coveragerc
index 67053a7..51038d6 100644
--- a/python/.coveragerc
+++ b/python/.coveragerc
@@ -2,4 +2,3 @@
 omit = tests/*
 source =
     sandcrawler
-    grobid2json
diff --git a/python/grobid2json.py b/python/grobid2json.py
deleted file mode 100755
index d92b351..0000000
--- a/python/grobid2json.py
+++ /dev/null
@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""
-NB: adapted to work as a library for PDF extraction. Will probably be
-re-written eventually to be correct, complete, and robust; this is just a
-first iteration.
-
-This script tries to extract everything from a GROBID TEI XML fulltext dump:
-
-- header metadata
-- affiliations
-- references (with context)
-- abstract
-- fulltext
-- tables, figures, equations
-
-A flag can be specified to disable copyright encumbered bits (--no-emcumbered):
-
-- abstract
-- fulltext
-- tables, figures, equations
-
-Prints JSON to stdout, errors to stderr
-"""
-
-import argparse
-import io
-import json
-import xml.etree.ElementTree as ET
-from typing import Any, AnyStr, Dict, List, Optional
-
-xml_ns = "http://www.w3.org/XML/1998/namespace"
-ns = "http://www.tei-c.org/ns/1.0"
-
-
-def all_authors(elem: Optional[ET.Element]) -> List[Dict[str, Any]]:
-    if not elem:
-        return []
-    names = []
-    for author in elem.findall(".//{%s}author" % ns):
-        pn = author.find("./{%s}persName" % ns)
-        if not pn:
-            continue
-        given_name = pn.findtext("./{%s}forename" % ns) or None
-        surname = pn.findtext("./{%s}surname" % ns) or None
-        full_name = " ".join(pn.itertext())
-        obj: Dict[str, Any] = dict(name=full_name)
-        if given_name:
-            obj["given_name"] = given_name
-        if surname:
-            obj["surname"] = surname
-        ae = author.find("./{%s}affiliation" % ns)
-        if ae:
-            affiliation: Dict[str, Any] = dict()
-            for on in ae.findall("./{%s}orgName" % ns):
-                on_type = on.get("type")
-                if on_type:
-                    affiliation[on_type] = on.text
-            addr_e = ae.find("./{%s}address" % ns)
-            if addr_e:
-                address = dict()
-                for t in addr_e:
-                    address[t.tag.split("}")[-1]] = t.text
-                if address:
-                    affiliation["address"] = address
-                # affiliation['address'] = {
-                #    'post_code': addr.findtext('./{%s}postCode' % ns) or None,
-                #    'settlement': addr.findtext('./{%s}settlement' % ns) or None,
-                #    'country': addr.findtext('./{%s}country' % ns) or None,
-                # }
-            obj["affiliation"] = affiliation
-        names.append(obj)
-    return names
-
-
-def journal_info(elem: ET.Element) -> Dict[str, Any]:
-    journal = dict()
-    journal["name"] = elem.findtext(".//{%s}monogr/{%s}title" % (ns, ns))
-    journal["publisher"] = elem.findtext(".//{%s}publicationStmt/{%s}publisher" % (ns, ns))
-    if journal["publisher"] == "":
-        journal["publisher"] = None
-    journal["issn"] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns)
-    journal["eissn"] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns)
-    journal["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
-    journal["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
-    keys = list(journal.keys())
-
-    # remove empty/null keys
-    for k in keys:
-        if not journal[k]:
-            journal.pop(k)
-    return journal
-
-
-def biblio_info(elem: ET.Element) -> Dict[str, Any]:
-    ref: Dict[str, Any] = dict()
-    ref["id"] = elem.attrib.get("{http://www.w3.org/XML/1998/namespace}id")
-    # Title stuff is messy in references...
-    ref["title"] = elem.findtext(".//{%s}analytic/{%s}title" % (ns, ns))
-    other_title = elem.findtext(".//{%s}monogr/{%s}title" % (ns, ns))
-    if other_title:
-        if ref["title"]:
-            ref["journal"] = other_title
-        else:
-            ref["journal"] = None
-            ref["title"] = other_title
-    ref["authors"] = all_authors(elem)
-    ref["publisher"] = elem.findtext(".//{%s}publicationStmt/{%s}publisher" % (ns, ns))
-    if ref["publisher"] == "":
-        ref["publisher"] = None
-    date = elem.find('.//{%s}date[@type="published"]' % ns)
-    ref["date"] = (date is not None) and date.attrib.get("when")
-    ref["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
-    ref["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
-    el = elem.find(".//{%s}ptr[@target]" % ns)
-    if el is not None:
-        ref["url"] = el.attrib["target"]
-        # Hand correction
-        if ref["url"].endswith(".Lastaccessed"):
-            ref["url"] = ref["url"].replace(".Lastaccessed", "")
-    else:
-        ref["url"] = None
-    return ref
-
-
-def teixml2json(content: AnyStr, encumbered: bool = True) -> Dict[str, Any]:
-
-    if isinstance(content, str):
-        tree = ET.parse(io.StringIO(content))
-    elif isinstance(content, bytes):
-        tree = ET.parse(io.BytesIO(content))
-
-    info: Dict[str, Any] = dict()
-
-    # print(content)
-    # print(content.getvalue())
-    tei = tree.getroot()
-
-    header = tei.find(".//{%s}teiHeader" % ns)
-    if header is None:
-        raise ValueError("XML does not look like TEI format")
-    application_tag = header.findall(".//{%s}appInfo/{%s}application" % (ns, ns))[0]
-    info["grobid_version"] = application_tag.attrib["version"].strip()
-    info["grobid_timestamp"] = application_tag.attrib["when"].strip()
-    info["title"] = header.findtext(".//{%s}analytic/{%s}title" % (ns, ns))
-    info["authors"] = all_authors(header.find(".//{%s}sourceDesc/{%s}biblStruct" % (ns, ns)))
-    info["journal"] = journal_info(header)
-    date = header.find('.//{%s}date[@type="published"]' % ns)
-    info["date"] = (date is not None) and date.attrib.get("when")
-    info["fatcat_release"] = header.findtext('.//{%s}idno[@type="fatcat"]' % ns)
-    info["doi"] = header.findtext('.//{%s}idno[@type="DOI"]' % ns)
-    if info["doi"]:
-        info["doi"] = info["doi"].lower()
-
-    refs = []
-    for (i, bs) in enumerate(tei.findall(".//{%s}listBibl/{%s}biblStruct" % (ns, ns))):
-        ref = biblio_info(bs)
-        ref["index"] = i
-        refs.append(ref)
-    info["citations"] = refs
-
-    text = tei.find(".//{%s}text" % (ns))
-    # print(text.attrib)
-    if text and text.attrib.get("{%s}lang" % xml_ns):
-        info["language_code"] = text.attrib["{%s}lang" % xml_ns]  # xml:lang
-
-    if encumbered:
-        el = tei.find(".//{%s}profileDesc/{%s}abstract" % (ns, ns))
-        info["abstract"] = (el or None) and " ".join(el.itertext()).strip()
-        el = tei.find(".//{%s}text/{%s}body" % (ns, ns))
-        info["body"] = (el or None) and " ".join(el.itertext()).strip()
-        el = tei.find('.//{%s}back/{%s}div[@type="acknowledgement"]' % (ns, ns))
-        info["acknowledgement"] = (el or None) and " ".join(el.itertext()).strip()
-        el = tei.find('.//{%s}back/{%s}div[@type="annex"]' % (ns, ns))
-        info["annex"] = (el or None) and " ".join(el.itertext()).strip()
-
-    # remove empty/null keys
-    keys = list(info.keys())
-    for k in keys:
-        if not info[k]:
-            info.pop(k)
-    return info
-
-
-def main() -> None:  # pragma no cover
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description="GROBID TEI XML to JSON",
-        usage="%(prog)s [options] <teifile>...",
-    )
-    parser.add_argument(
-        "--no-encumbered",
-        action="store_true",
-        help="don't include ambiguously copyright encumbered fields (eg, abstract, body)",
-    )
-    parser.add_argument("teifiles", nargs="+")
-
-    args = parser.parse_args()
-
-    for filename in args.teifiles:
-        content = open(filename, "r").read()
-        print(
-            json.dumps(
-                teixml2json(content, encumbered=(not args.no_encumbered)),
-                sort_keys=True,
-            )
-        )
-
-
-if __name__ == "__main__":  # pragma no cover
-    main()
diff --git a/python/grobid_tool.py b/python/grobid_tool.py
index f85d243..f99a78b 100755
--- a/python/grobid_tool.py
+++ b/python/grobid_tool.py
@@ -12,7 +12,8 @@ import argparse
 import json
 import sys
 
-from grobid2json import teixml2json
+from grobid_tei_xml import parse_document_xml
+
 from sandcrawler import *
 
 
@@ -75,7 +76,8 @@ def run_transform(args):
         if args.metadata_only:
             out = grobid_client.metadata(line)
         else:
-            out = teixml2json(line["tei_xml"])
+            tei_doc = parse_document_xml(line["tei_xml"])
+            out = tei_doc.to_legacy_dict()
         if out:
             if "source" in line:
                 out["source"] = line["source"]
diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py
index 26918f6..37c4ea1 100644
--- a/python/sandcrawler/grobid.py
+++ b/python/sandcrawler/grobid.py
@@ -1,8 +1,7 @@
 from typing import Any, Dict, Optional
 
 import requests
-
-from grobid2json import teixml2json
+from grobid_tei_xml import parse_document_xml
 
 from .ia import WaybackClient
 from .misc import gen_file_metadata
@@ -71,7 +70,9 @@ class GrobidClient(object):
     def metadata(self, result: Dict[str, Any]) -> Optional[Dict[str, Any]]:
         if result["status"] != "success":
             return None
-        tei_json = teixml2json(result["tei_xml"], encumbered=False)
+        tei_doc = parse_document_xml(result["tei_xml"])
+        tei_doc.remove_encumbered()
+        tei_json = tei_doc.to_legacy_dict()
         meta = dict()
         biblio = dict()
         for k in (
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py
index c8c0c33..f50b9d1 100644
--- a/python/sandcrawler/persist.py
+++ b/python/sandcrawler/persist.py
@@ -395,7 +395,7 @@ class PersistGrobidWorker(SandcrawlerWorker):
                 )
                 self.counts["s3-put"] += 1
 
-            # enhance with teixml2json metadata, if available
+            # enhance with GROBID TEI-XML metadata, if available
             try:
                 metadata = self.grobid.metadata(r)
             except xml.etree.ElementTree.ParseError as xml_e:
diff --git a/python/scripts/grobid_affiliations.py b/python/scripts/grobid_affiliations.py
index b01e46a..90a0f77 100755
--- a/python/scripts/grobid_affiliations.py
+++ b/python/scripts/grobid_affiliations.py
@@ -12,7 +12,7 @@ Run in bulk like:
 import json
 import sys
 
-from grobid2json import teixml2json
+from grobid_tei_xml import parse_document_xml
 
 
 def parse_hbase(line):
@@ -38,7 +38,9 @@ def run(mode="hbase"):
         else:
             raise NotImplementedError("parse mode: {}".format(mode))
 
-        obj = teixml2json(tei_xml, encumbered=False)
+        tei_doc = parse_document_xml(tei_xml)
+        tei_doc.remove_encumbered()
+        obj = tei_doc.to_legacy_dict()
 
         affiliations = []
         for author in obj["authors"]:
diff --git a/python/tests/test_grobid2json.py b/python/tests/test_grobid2json.py
index 98888e8..b00a88d 100644
--- a/python/tests/test_grobid2json.py
+++ b/python/tests/test_grobid2json.py
@@ -2,23 +2,27 @@ import json
 import xml
 
 import pytest
-
-from grobid2json import *
+from grobid_tei_xml import parse_document_xml
 
 
 def test_small_xml():
+    """
+    This used to be a test of grobid2json; now it is a compatability test for
+    the to_legacy_dict() feature of grobid_tei_xml.
+    """
 
     with open("tests/files/small.xml", "r") as f:
         tei_xml = f.read()
     with open("tests/files/small.json", "r") as f:
         json_form = json.loads(f.read())
 
-    assert teixml2json(tei_xml) == json_form
+    tei_doc = parse_document_xml(tei_xml)
+    assert tei_doc.to_legacy_dict() == json_form
 
 
 def test_invalid_xml():
 
     with pytest.raises(xml.etree.ElementTree.ParseError):
-        teixml2json("this is not XML")
+        parse_document_xml("this is not XML")
     with pytest.raises(ValueError):
-        teixml2json("<xml></xml>")
+        parse_document_xml("<xml></xml>")