From a0e275a4bad46ef41585f0207d6dfa1e3c38bc35 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 27 Oct 2021 19:10:35 -0700 Subject: remove grobid2json helper file, replace with grobid_tei_xml --- python/.coveragerc | 1 - python/grobid2json.py | 210 ---------------------------------- python/grobid_tool.py | 6 +- python/sandcrawler/grobid.py | 7 +- python/sandcrawler/persist.py | 2 +- python/scripts/grobid_affiliations.py | 6 +- python/tests/test_grobid2json.py | 14 ++- 7 files changed, 22 insertions(+), 224 deletions(-) delete mode 100755 python/grobid2json.py (limited to 'python') diff --git a/python/.coveragerc b/python/.coveragerc index 67053a7..51038d6 100644 --- a/python/.coveragerc +++ b/python/.coveragerc @@ -2,4 +2,3 @@ omit = tests/* source = sandcrawler - grobid2json diff --git a/python/grobid2json.py b/python/grobid2json.py deleted file mode 100755 index d92b351..0000000 --- a/python/grobid2json.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env python3 -""" -NB: adapted to work as a library for PDF extraction. Will probably be -re-written eventually to be correct, complete, and robust; this is just a -first iteration. - -This script tries to extract everything from a GROBID TEI XML fulltext dump: - -- header metadata -- affiliations -- references (with context) -- abstract -- fulltext -- tables, figures, equations - -A flag can be specified to disable copyright encumbered bits (--no-emcumbered): - -- abstract -- fulltext -- tables, figures, equations - -Prints JSON to stdout, errors to stderr -""" - -import argparse -import io -import json -import xml.etree.ElementTree as ET -from typing import Any, AnyStr, Dict, List, Optional - -xml_ns = "http://www.w3.org/XML/1998/namespace" -ns = "http://www.tei-c.org/ns/1.0" - - -def all_authors(elem: Optional[ET.Element]) -> List[Dict[str, Any]]: - if not elem: - return [] - names = [] - for author in elem.findall(".//{%s}author" % ns): - pn = author.find("./{%s}persName" % ns) - if not pn: - continue - given_name = pn.findtext("./{%s}forename" % ns) or None - surname = pn.findtext("./{%s}surname" % ns) or None - full_name = " ".join(pn.itertext()) - obj: Dict[str, Any] = dict(name=full_name) - if given_name: - obj["given_name"] = given_name - if surname: - obj["surname"] = surname - ae = author.find("./{%s}affiliation" % ns) - if ae: - affiliation: Dict[str, Any] = dict() - for on in ae.findall("./{%s}orgName" % ns): - on_type = on.get("type") - if on_type: - affiliation[on_type] = on.text - addr_e = ae.find("./{%s}address" % ns) - if addr_e: - address = dict() - for t in addr_e: - address[t.tag.split("}")[-1]] = t.text - if address: - affiliation["address"] = address - # affiliation['address'] = { - # 'post_code': addr.findtext('./{%s}postCode' % ns) or None, - # 'settlement': addr.findtext('./{%s}settlement' % ns) or None, - # 'country': addr.findtext('./{%s}country' % ns) or None, - # } - obj["affiliation"] = affiliation - names.append(obj) - return names - - -def journal_info(elem: ET.Element) -> Dict[str, Any]: - journal = dict() - journal["name"] = elem.findtext(".//{%s}monogr/{%s}title" % (ns, ns)) - journal["publisher"] = elem.findtext(".//{%s}publicationStmt/{%s}publisher" % (ns, ns)) - if journal["publisher"] == "": - journal["publisher"] = None - journal["issn"] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns) - journal["eissn"] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns) - journal["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) - journal["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) - keys = list(journal.keys()) - - # remove empty/null keys - for k in keys: - if not journal[k]: - journal.pop(k) - return journal - - -def biblio_info(elem: ET.Element) -> Dict[str, Any]: - ref: Dict[str, Any] = dict() - ref["id"] = elem.attrib.get("{http://www.w3.org/XML/1998/namespace}id") - # Title stuff is messy in references... - ref["title"] = elem.findtext(".//{%s}analytic/{%s}title" % (ns, ns)) - other_title = elem.findtext(".//{%s}monogr/{%s}title" % (ns, ns)) - if other_title: - if ref["title"]: - ref["journal"] = other_title - else: - ref["journal"] = None - ref["title"] = other_title - ref["authors"] = all_authors(elem) - ref["publisher"] = elem.findtext(".//{%s}publicationStmt/{%s}publisher" % (ns, ns)) - if ref["publisher"] == "": - ref["publisher"] = None - date = elem.find('.//{%s}date[@type="published"]' % ns) - ref["date"] = (date is not None) and date.attrib.get("when") - ref["volume"] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns) - ref["issue"] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns) - el = elem.find(".//{%s}ptr[@target]" % ns) - if el is not None: - ref["url"] = el.attrib["target"] - # Hand correction - if ref["url"].endswith(".Lastaccessed"): - ref["url"] = ref["url"].replace(".Lastaccessed", "") - else: - ref["url"] = None - return ref - - -def teixml2json(content: AnyStr, encumbered: bool = True) -> Dict[str, Any]: - - if isinstance(content, str): - tree = ET.parse(io.StringIO(content)) - elif isinstance(content, bytes): - tree = ET.parse(io.BytesIO(content)) - - info: Dict[str, Any] = dict() - - # print(content) - # print(content.getvalue()) - tei = tree.getroot() - - header = tei.find(".//{%s}teiHeader" % ns) - if header is None: - raise ValueError("XML does not look like TEI format") - application_tag = header.findall(".//{%s}appInfo/{%s}application" % (ns, ns))[0] - info["grobid_version"] = application_tag.attrib["version"].strip() - info["grobid_timestamp"] = application_tag.attrib["when"].strip() - info["title"] = header.findtext(".//{%s}analytic/{%s}title" % (ns, ns)) - info["authors"] = all_authors(header.find(".//{%s}sourceDesc/{%s}biblStruct" % (ns, ns))) - info["journal"] = journal_info(header) - date = header.find('.//{%s}date[@type="published"]' % ns) - info["date"] = (date is not None) and date.attrib.get("when") - info["fatcat_release"] = header.findtext('.//{%s}idno[@type="fatcat"]' % ns) - info["doi"] = header.findtext('.//{%s}idno[@type="DOI"]' % ns) - if info["doi"]: - info["doi"] = info["doi"].lower() - - refs = [] - for (i, bs) in enumerate(tei.findall(".//{%s}listBibl/{%s}biblStruct" % (ns, ns))): - ref = biblio_info(bs) - ref["index"] = i - refs.append(ref) - info["citations"] = refs - - text = tei.find(".//{%s}text" % (ns)) - # print(text.attrib) - if text and text.attrib.get("{%s}lang" % xml_ns): - info["language_code"] = text.attrib["{%s}lang" % xml_ns] # xml:lang - - if encumbered: - el = tei.find(".//{%s}profileDesc/{%s}abstract" % (ns, ns)) - info["abstract"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find(".//{%s}text/{%s}body" % (ns, ns)) - info["body"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find('.//{%s}back/{%s}div[@type="acknowledgement"]' % (ns, ns)) - info["acknowledgement"] = (el or None) and " ".join(el.itertext()).strip() - el = tei.find('.//{%s}back/{%s}div[@type="annex"]' % (ns, ns)) - info["annex"] = (el or None) and " ".join(el.itertext()).strip() - - # remove empty/null keys - keys = list(info.keys()) - for k in keys: - if not info[k]: - info.pop(k) - return info - - -def main() -> None: # pragma no cover - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="GROBID TEI XML to JSON", - usage="%(prog)s [options] ...", - ) - parser.add_argument( - "--no-encumbered", - action="store_true", - help="don't include ambiguously copyright encumbered fields (eg, abstract, body)", - ) - parser.add_argument("teifiles", nargs="+") - - args = parser.parse_args() - - for filename in args.teifiles: - content = open(filename, "r").read() - print( - json.dumps( - teixml2json(content, encumbered=(not args.no_encumbered)), - sort_keys=True, - ) - ) - - -if __name__ == "__main__": # pragma no cover - main() diff --git a/python/grobid_tool.py b/python/grobid_tool.py index f85d243..f99a78b 100755 --- a/python/grobid_tool.py +++ b/python/grobid_tool.py @@ -12,7 +12,8 @@ import argparse import json import sys -from grobid2json import teixml2json +from grobid_tei_xml import parse_document_xml + from sandcrawler import * @@ -75,7 +76,8 @@ def run_transform(args): if args.metadata_only: out = grobid_client.metadata(line) else: - out = teixml2json(line["tei_xml"]) + tei_doc = parse_document_xml(line["tei_xml"]) + out = tei_doc.to_legacy_dict() if out: if "source" in line: out["source"] = line["source"] diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py index 26918f6..37c4ea1 100644 --- a/python/sandcrawler/grobid.py +++ b/python/sandcrawler/grobid.py @@ -1,8 +1,7 @@ from typing import Any, Dict, Optional import requests - -from grobid2json import teixml2json +from grobid_tei_xml import parse_document_xml from .ia import WaybackClient from .misc import gen_file_metadata @@ -71,7 +70,9 @@ class GrobidClient(object): def metadata(self, result: Dict[str, Any]) -> Optional[Dict[str, Any]]: if result["status"] != "success": return None - tei_json = teixml2json(result["tei_xml"], encumbered=False) + tei_doc = parse_document_xml(result["tei_xml"]) + tei_doc.remove_encumbered() + tei_json = tei_doc.to_legacy_dict() meta = dict() biblio = dict() for k in ( diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index c8c0c33..f50b9d1 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -395,7 +395,7 @@ class PersistGrobidWorker(SandcrawlerWorker): ) self.counts["s3-put"] += 1 - # enhance with teixml2json metadata, if available + # enhance with GROBID TEI-XML metadata, if available try: metadata = self.grobid.metadata(r) except xml.etree.ElementTree.ParseError as xml_e: diff --git a/python/scripts/grobid_affiliations.py b/python/scripts/grobid_affiliations.py index b01e46a..90a0f77 100755 --- a/python/scripts/grobid_affiliations.py +++ b/python/scripts/grobid_affiliations.py @@ -12,7 +12,7 @@ Run in bulk like: import json import sys -from grobid2json import teixml2json +from grobid_tei_xml import parse_document_xml def parse_hbase(line): @@ -38,7 +38,9 @@ def run(mode="hbase"): else: raise NotImplementedError("parse mode: {}".format(mode)) - obj = teixml2json(tei_xml, encumbered=False) + tei_doc = parse_document_xml(tei_xml) + tei_doc.remove_encumbered() + obj = tei_doc.to_legacy_dict() affiliations = [] for author in obj["authors"]: diff --git a/python/tests/test_grobid2json.py b/python/tests/test_grobid2json.py index 98888e8..b00a88d 100644 --- a/python/tests/test_grobid2json.py +++ b/python/tests/test_grobid2json.py @@ -2,23 +2,27 @@ import json import xml import pytest - -from grobid2json import * +from grobid_tei_xml import parse_document_xml def test_small_xml(): + """ + This used to be a test of grobid2json; now it is a compatability test for + the to_legacy_dict() feature of grobid_tei_xml. + """ with open("tests/files/small.xml", "r") as f: tei_xml = f.read() with open("tests/files/small.json", "r") as f: json_form = json.loads(f.read()) - assert teixml2json(tei_xml) == json_form + tei_doc = parse_document_xml(tei_xml) + assert tei_doc.to_legacy_dict() == json_form def test_invalid_xml(): with pytest.raises(xml.etree.ElementTree.ParseError): - teixml2json("this is not XML") + parse_document_xml("this is not XML") with pytest.raises(ValueError): - teixml2json("") + parse_document_xml("") -- cgit v1.2.3