aboutsummaryrefslogtreecommitdiffstats
path: root/python/scripts
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2019-10-02 19:10:21 -0700
committerBryan Newbold <bnewbold@archive.org>2019-10-02 19:10:21 -0700
commit1c3e95612d56681fede45f8c8ec3991f9cb9ac1d (patch)
tree6ac1bdf5b4579f0eab786e8d74e8a33f0eddd899 /python/scripts
parent79ebb924dbd3c724ecfbe2b00e18f0ae7e51d4a6 (diff)
downloadsandcrawler-1c3e95612d56681fede45f8c8ec3991f9cb9ac1d.tar.gz
sandcrawler-1c3e95612d56681fede45f8c8ec3991f9cb9ac1d.zip
grobid_affiliations fix from prod, and usage example
Diffstat (limited to 'python/scripts')
-rwxr-xr-xpython/scripts/grobid_affiliations.py5
1 files changed, 5 insertions, 0 deletions
diff --git a/python/scripts/grobid_affiliations.py b/python/scripts/grobid_affiliations.py
index 2b1d29e..79feac1 100755
--- a/python/scripts/grobid_affiliations.py
+++ b/python/scripts/grobid_affiliations.py
@@ -4,6 +4,10 @@
Takes old (HBase) or new (pg) style JSON wrappers of GROBID XML extraction
output, converts the XML to JSON, filters out raw affiliation strings, and
dumps these as JSON subset.
+
+Run in bulk like:
+
+ ls /bigger/unpaywall-transfer/2019-07-17-1741.30-dumpgrobidxml/part*gz | parallel --progress -j8 'zcat {} | ./grobid_affiliations.py > {}.affiliations'
"""
import sys
@@ -41,6 +45,7 @@ def run(mode='hbase'):
if affiliations:
# don't duplicate affiliations; only the unique ones
affiliations = list(set([json.dumps(a) for a in affiliations]))
+ affiliations = [json.loads(a) for a in affiliations]
print('\t'.join([sha1hex, json.dumps(affiliations)]))
if __name__=='__main__':