1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#!/usr/bin/env python3
"""
NB: adapted to work as a library for PDF extraction. Will probably be
re-written eventually to be correct, complete, and robust; this is just a
first iteration.
This script tries to extract everything from a GROBID TEI XML fulltext dump:
- header metadata
- affiliations
- references (with context)
- abstract
- fulltext
- tables, figures, equations
A flag can be specified to disable copyright encumbered bits (--no-emcumbered):
- abstract
- fulltext
- tables, figures, equations
Prints JSON to stdout, errors to stderr
"""
import io
import json
import argparse
import xml.etree.ElementTree as ET
ns = "http://www.tei-c.org/ns/1.0"
def all_authors(elem):
names = [' '.join([e.findtext('./{%s}forename' % ns) or '', e.findtext('./{%s}surname' % ns) or '']).strip()
for e in elem.findall('.//{%s}author/{%s}persName' % (ns, ns))]
return [dict(name=n) for n in names]
def journal_info(elem):
journal = dict()
journal['name'] = elem.findtext('.//{%s}monogr/{%s}title' % (ns, ns))
journal['publisher'] = elem.findtext('.//{%s}publicationStmt/{%s}publisher' % (ns, ns))
if journal['publisher'] == '':
journal['publisher'] = None
journal['issn'] = elem.findtext('.//{%s}idno[@type="ISSN"]' % ns)
journal['eissn'] = elem.findtext('.//{%s}idno[@type="eISSN"]' % ns)
journal['volume'] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
journal['issue'] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
return journal
def biblio_info(elem):
ref = dict()
ref['id'] = elem.attrib.get('{http://www.w3.org/XML/1998/namespace}id')
# Title stuff is messy in references...
ref['title'] = elem.findtext('.//{%s}analytic/{%s}title' % (ns, ns))
other_title = elem.findtext('.//{%s}monogr/{%s}title' % (ns, ns))
if other_title:
if ref['title']:
ref['journal'] = other_title
else:
ref['journal'] = None
ref['title'] = other_title
ref['authors'] = all_authors(elem)
ref['publisher'] = elem.findtext('.//{%s}publicationStmt/{%s}publisher' % (ns, ns))
if ref['publisher'] == '':
ref['publisher'] = None
date = elem.find('.//{%s}date[@type="published"]' % ns)
ref['date'] = (date != None) and date.attrib.get('when')
ref['volume'] = elem.findtext('.//{%s}biblScope[@unit="volume"]' % ns)
ref['issue'] = elem.findtext('.//{%s}biblScope[@unit="issue"]' % ns)
el = elem.find('.//{%s}ptr[@target]' % ns)
if el is not None:
ref['url'] = el.attrib['target']
# Hand correction
if ref['url'].endswith(".Lastaccessed"):
ref['url'] = ref['url'].replace(".Lastaccessed", "")
else:
ref['url'] = None
return ref
def teixml2json(content, encumbered=True):
if type(content) == str:
content = io.StringIO(content)
elif type(content) == bytes:
content = io.BytesIO(content)
info = dict()
#print(content)
#print(content.getvalue())
tree = ET.parse(content)
tei = tree.getroot()
header = tei.find('.//{%s}teiHeader' % ns)
if header is None:
raise ValueError("XML does not look like TEI format")
info['title'] = header.findtext('.//{%s}analytic/{%s}title' % (ns, ns))
info['authors'] = all_authors(header.find('.//{%s}sourceDesc/{%s}biblStruct' % (ns, ns)))
info['journal'] = journal_info(header)
date = header.find('.//{%s}date[@type="published"]' % ns)
info['date'] = (date != None) and date.attrib.get('when')
info['doi'] = header.findtext('.//{%s}idno[@type="DOI"]' % ns)
if info['doi']:
info['doi'] = info['doi'].lower()
refs = []
for (i, bs) in enumerate(tei.findall('.//{%s}listBibl/{%s}biblStruct' % (ns, ns))):
ref = biblio_info(bs)
ref['index'] = i
refs.append(ref)
info['citations'] = refs
if encumbered:
el = tei.find('.//{%s}profileDesc/{%s}abstract' % (ns, ns))
info['abstract'] = (el or None) and " ".join(el.itertext()).strip()
el = tei.find('.//{%s}text/{%s}body' % (ns, ns))
info['body'] = (el or None) and " ".join(el.itertext()).strip()
el = tei.find('.//{%s}back/{%s}div[@type="acknowledgement"]' % (ns, ns))
info['acknowledgement'] = (el or None) and " ".join(el.itertext()).strip()
el = tei.find('.//{%s}back/{%s}div[@type="annex"]' % (ns, ns))
info['annex'] = (el or None) and " ".join(el.itertext()).strip()
return info
def main(): # pragma no cover
parser = argparse.ArgumentParser(
description="GROBID TEI XML to JSON",
usage="%(prog)s [options] <teifile>...")
parser.add_argument("--no-encumbered",
action="store_true",
help="ignore errors loading individual WARC files")
parser.add_argument("teifiles", nargs='+')
args = parser.parse_args()
for filename in args.teifiles:
content = open(filename, 'r')
print(json.dumps(
teixml2json(content,
encumbered=(not args.no_encumbered))))
if __name__=='__main__': # pragma no cover
main()
|
