From b362abd38ad4a6624bc056c58eb90ae235c63f00 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 23 Sep 2019 23:00:23 -0700 Subject: rename postgrest directory sql --- postgrest/backfill/backfill_grobid_unpaywall.py | 59 ------------------------- 1 file changed, 59 deletions(-) delete mode 100755 postgrest/backfill/backfill_grobid_unpaywall.py (limited to 'postgrest/backfill/backfill_grobid_unpaywall.py') diff --git a/postgrest/backfill/backfill_grobid_unpaywall.py b/postgrest/backfill/backfill_grobid_unpaywall.py deleted file mode 100755 index 58e9e3c..0000000 --- a/postgrest/backfill/backfill_grobid_unpaywall.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -""" -This is a "one-time" tranform helper script for GROBID backfill into -sandcrawler minio and postgresql. - -This variant of backfill_grobid.py pushes into the unpaywall bucket of -sandcrawler-minio and doesn't push anything to sandcrawler table in general. -""" - -import json, os, sys, collections, io -import base64 -import requests -from minio import Minio -import psycopg2 -import psycopg2.extras - - -def b32_hex(s): - s = s.strip().split()[0].lower() - if s.startswith("sha1:"): - s = s[5:] - if len(s) != 32: - return s - return base64.b16encode(base64.b32decode(s.upper())).lower().decode('utf-8') - -def stdin_to_minio(): - mc = Minio('localhost:9000', - access_key=os.environ['MINIO_ACCESS_KEY'], - secret_key=os.environ['MINIO_SECRET_KEY'], - secure=False) - counts = collections.Counter({'total': 0}) - for l in sys.stdin: - if counts['raw_lines'] > 0 and counts['raw_lines'] % 10000 == 0: - print("Progress: {}...".format(counts)) - counts['raw_lines'] += 1 - l = l.strip() - if not l: - continue - row = json.loads(l) - if not row: - continue - sha1hex = b32_hex(row['pdf_hash']) - grobid_xml = row['tei_xml'].encode('utf-8') - grobid_xml_len = len(grobid_xml) - grobid_xml = io.BytesIO(grobid_xml) - - key = "grobid/{}/{}/{}.tei.xml".format( - sha1hex[0:2], - sha1hex[2:4], - sha1hex) - mc.put_object("unpaywall", key, grobid_xml, grobid_xml_len, - content_type="application/tei+xml", - metadata=None) - counts['minio-success'] += 1 - - print("Done: {}".format(counts)) - -if __name__=='__main__': - stdin_to_minio() -- cgit v1.2.3