diff options
author | Bryan Newbold <bnewbold@archive.org> | 2019-09-23 23:00:23 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2019-09-23 23:00:23 -0700 |
commit | b362abd38ad4a6624bc056c58eb90ae235c63f00 (patch) | |
tree | 026556fe548e28be1556c24b6ee865cb276755ca /sql/backfill/backfill_grobid_unpaywall.py | |
parent | b438f52dbb7578c9a5c2153bc4ba50e33fdae7c3 (diff) | |
download | sandcrawler-b362abd38ad4a6624bc056c58eb90ae235c63f00.tar.gz sandcrawler-b362abd38ad4a6624bc056c58eb90ae235c63f00.zip |
rename postgrest directory sql
Diffstat (limited to 'sql/backfill/backfill_grobid_unpaywall.py')
-rwxr-xr-x | sql/backfill/backfill_grobid_unpaywall.py | 59 |
1 files changed, 59 insertions, 0 deletions
diff --git a/sql/backfill/backfill_grobid_unpaywall.py b/sql/backfill/backfill_grobid_unpaywall.py new file mode 100755 index 0000000..58e9e3c --- /dev/null +++ b/sql/backfill/backfill_grobid_unpaywall.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +This is a "one-time" tranform helper script for GROBID backfill into +sandcrawler minio and postgresql. + +This variant of backfill_grobid.py pushes into the unpaywall bucket of +sandcrawler-minio and doesn't push anything to sandcrawler table in general. +""" + +import json, os, sys, collections, io +import base64 +import requests +from minio import Minio +import psycopg2 +import psycopg2.extras + + +def b32_hex(s): + s = s.strip().split()[0].lower() + if s.startswith("sha1:"): + s = s[5:] + if len(s) != 32: + return s + return base64.b16encode(base64.b32decode(s.upper())).lower().decode('utf-8') + +def stdin_to_minio(): + mc = Minio('localhost:9000', + access_key=os.environ['MINIO_ACCESS_KEY'], + secret_key=os.environ['MINIO_SECRET_KEY'], + secure=False) + counts = collections.Counter({'total': 0}) + for l in sys.stdin: + if counts['raw_lines'] > 0 and counts['raw_lines'] % 10000 == 0: + print("Progress: {}...".format(counts)) + counts['raw_lines'] += 1 + l = l.strip() + if not l: + continue + row = json.loads(l) + if not row: + continue + sha1hex = b32_hex(row['pdf_hash']) + grobid_xml = row['tei_xml'].encode('utf-8') + grobid_xml_len = len(grobid_xml) + grobid_xml = io.BytesIO(grobid_xml) + + key = "grobid/{}/{}/{}.tei.xml".format( + sha1hex[0:2], + sha1hex[2:4], + sha1hex) + mc.put_object("unpaywall", key, grobid_xml, grobid_xml_len, + content_type="application/tei+xml", + metadata=None) + counts['minio-success'] += 1 + + print("Done: {}".format(counts)) + +if __name__=='__main__': + stdin_to_minio() |