1 files changed, 146 insertions, 0 deletions
diff --git a/python/grobid2json.py b/python/grobid2json.py
new file mode 100755
index 0000000..ca460f8
--- /dev/null
+++ b/python/grobid2json.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+"""
+NB: adapted to work as a library for PDF extraction. Will probably be
+re-written eventually to be correct, complete, and robust; this is just a
+first iteration.
+
+This script tries to extract everything from a GROBID TEI XML fulltext dump:
+
+- header metadata
+- affiliations
+- references (with context)
+- abstract
+- fulltext
+- tables, figures, equations
+
+A flag can be specified to disable copyright encumbered bits (--no-emcumbered):
+
+- abstract
+- fulltext
+- tables, figures, equations
+
+Prints JSON to stdout, errors to stderr
+"""
+
+import io
+import json
+import argparse
+import xml.etree.ElementTree as ET
+
+ns = "http://www.tei-c.org/ns/1.0"
+
+def all_authors(elem):
+    names = [' '.join([e.findtext('./{%s}forename' % ns) or '', e.findtext('./{%s}surname' % ns) or '']).strip()
+            for e in elem.findall('.//{%s}author/{%s}persName' % (ns, ns))]
+    return [dict(name=n) for n in names]
+
+
+def journal_info(elem):
+    journal = dict()
+    journal['name'] = elem.findtext('.//{%s}monogr/{%s}title' % (ns, ns))
+    journal['publisher'] = elem.findtext('.//{%s}publicationStmt/{%s}publisher' % (ns, ns))
+    if journal['publisher'] == '':
+        journal['publisher'] = None
+    journal['issn'] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns)
+    journal['eissn'] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns)
+    journal['volume'] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
+    journal['issue'] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
+    return journal
+
+
+def biblio_info(elem):
+    ref = dict()
+    ref['id'] = elem.attrib.get('{http://www.w3.org/XML/1998/namespace}id')
+    # Title stuff is messy in references...
+    ref['title'] = elem.findtext('.//{%s}analytic/{%s}title' % (ns, ns))
+    other_title = elem.findtext('.//{%s}monogr/{%s}title' % (ns, ns))
+    if other_title:
+        if ref['title']:
+            ref['journal'] = other_title
+        else:
+            ref['journal'] = None
+            ref['title'] = other_title
+    ref['authors'] = all_authors(elem)
+    ref['publisher'] = elem.findtext('.//{%s}publicationStmt/{%s}publisher' % (ns, ns))
+    if ref['publisher'] == '':
+        ref['publisher'] = None
+    date = elem.find('.//{%s}date[@type="published"]' % ns)
+    ref['date'] = (date != None) and date.attrib.get('when')
+    ref['volume'] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
+    ref['issue'] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
+    el = elem.find('.//{%s}ptr[@target]' % ns)
+    if el is not None:
+        ref['url'] = el.attrib['target']
+        # Hand correction
+        if ref['url'].endswith(".Lastaccessed"):
+            ref['url'] = ref['url'].replace(".Lastaccessed", "")
+    else:
+        ref['url'] = None
+    return ref
+
+
+def teixml2json(content, encumbered=True):
+
+    if type(content) == str:
+        content = io.StringIO(content)
+    elif type(content) == bytes:
+        content = io.BytesIO(content)
+
+    info = dict()
+
+    #print(content)
+    #print(content.getvalue())
+    tree = ET.parse(content)
+    tei = tree.getroot()
+
+    header = tei.find('.//{%s}teiHeader' % ns)
+    if header is None:
+        raise ValueError("XML does not look like TEI format")
+    info['title'] = header.findtext('.//{%s}analytic/{%s}title' % (ns, ns))
+    info['authors'] = all_authors(header.find('.//{%s}sourceDesc/{%s}biblStruct' % (ns, ns)))
+    info['journal'] = journal_info(header)
+    date = header.find('.//{%s}date[@type="published"]' % ns)
+    info['date'] = (date != None) and date.attrib.get('when')
+    info['doi'] = header.findtext('.//{%s}idno[@type="DOI"]' % ns)
+    if info['doi']:
+        info['doi'] = info['doi'].lower()
+
+    refs = []
+    for (i, bs) in enumerate(tei.findall('.//{%s}listBibl/{%s}biblStruct' % (ns, ns))):
+        ref = biblio_info(bs)
+        ref['index'] = i
+        refs.append(ref)
+    info['citations'] = refs
+
+    if encumbered:
+        el = tei.find('.//{%s}profileDesc/{%s}abstract' % (ns, ns))
+        info['abstract'] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find('.//{%s}text/{%s}body' % (ns, ns))
+        info['body'] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find('.//{%s}back/{%s}div[@type="acknowledgement"]' % (ns, ns))
+        info['acknowledgement'] = (el or None) and " ".join(el.itertext()).strip()
+        el = tei.find('.//{%s}back/{%s}div[@type="annex"]' % (ns, ns))
+        info['annex'] = (el or None) and " ".join(el.itertext()).strip()
+
+    return info
+
+def main():   # pragma no cover
+    parser = argparse.ArgumentParser(
+        description="GROBID TEI XML to JSON",
+        usage="%(prog)s [options] <teifile>...")
+    parser.add_argument("--no-encumbered",
+        action="store_true",
+        help="ignore errors loading individual WARC files")
+    parser.add_argument("teifiles", nargs='+')
+
+    args = parser.parse_args()
+
+    for filename in args.teifiles:
+        content = open(filename, 'r')
+        print(json.dumps(
+            teixml2json(content,
+               encumbered=(not args.no_encumbered))))
+
+if __name__=='__main__':   # pragma no cover
+    main()