diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/cord19_fatcat_enrich.py | 104 | ||||
| -rwxr-xr-x | scripts/deliver_file2disk.py | 225 | ||||
| -rwxr-xr-x | scripts/parse_cord19_csv.py | 15 |
3 files changed, 0 insertions, 344 deletions
diff --git a/scripts/cord19_fatcat_enrich.py b/scripts/cord19_fatcat_enrich.py deleted file mode 100755 index 5d3a554..0000000 --- a/scripts/cord19_fatcat_enrich.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -""" -Takes a JSON-transformed CORD-19 *metadata* file and enriches it with fatcat -metadata. -""" - -import sys -import json -import argparse -import datetime - -import requests -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error - - -def requests_retry_session(retries=10, backoff_factor=3, - status_forcelist=(500, 502, 504), session=None): - """ - From: https://www.peterbe.com/plog/best-practice-with-retries-with-requests - """ - session = session or requests.Session() - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=backoff_factor, - status_forcelist=status_forcelist, - ) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - return session - - -def do_line(row, args): - - pubmed_id = row.get('pubmed_id') or None - pmcid = row.get('pmcid') or None - doi = row.get('doi') or None - fatcat_release = None - - if doi == '0.1126/science.abb7331': - doi = '10.1126/science.abb7331' - - if not fatcat_release and pmcid: - resp = args.session.get('https://api.fatcat.wiki/v0/release/lookup', - params={ - 'pmcid': pmcid, - 'expand': 'container,files,filesets,webcaptures', - 'hide': 'abstracts,references', - }) - if resp.status_code == 200: - fatcat_release = resp.json() - if not fatcat_release and doi: - resp = args.session.get('https://api.fatcat.wiki/v0/release/lookup', - params={ - 'doi': doi, - 'expand': 'container,files,filesets,webcaptures', - 'hide': 'abstracts,references', - }) - if resp.status_code == 200: - fatcat_release = resp.json() - if not fatcat_release and pubmed_id: - resp = args.session.get('https://api.fatcat.wiki/v0/release/lookup', - params={ - 'pmid': pubmed_id, - 'expand': 'container,files,filesets,webcaptures', - 'hide': 'abstracts,references', - }) - if resp.status_code == 200: - fatcat_release = resp.json() - - obj = dict( - cord19_paper=row, - ) - if fatcat_release: - obj['fatcat_release'] = fatcat_release - obj['release_id'] = fatcat_release['ident'] - obj['fatcat_url'] = "https://fatcat.wiki/release/{}".format(obj['release_id']) - print(json.dumps(obj, sort_keys=True)) - -def run(args): - for l in sys.stdin: - l = json.loads(l) - do_line(l, args) - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('json_file', - help="CORD-19 parsed JSON file", - type=argparse.FileType('r')) - subparsers = parser.add_subparsers() - - args = parser.parse_args() - args.session = requests_retry_session() - - run(args) - -if __name__ == '__main__': - main() - diff --git a/scripts/deliver_file2disk.py b/scripts/deliver_file2disk.py deleted file mode 100755 index f54ecb3..0000000 --- a/scripts/deliver_file2disk.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -""" -Tool for downloading fatcat release PDFs to disk (assuming there is at least -one accessible PDF file entity for each release). - -Behavior: -- if no file, or not accessible, skip release -- filter files, then iterate through: - - if already exists locally on disk, skip - - try downloading from any archive.org or web.archive.org URLs - - verify SHA-1 - - write out to disk -""" - -# XXX: some broken MRO thing going on in here due to python3 object wrangling -# in `wayback` library. Means we can't run pylint. -# pylint: skip-file - -import os -import sys -import json -import magic -import hashlib -import argparse -import requests -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error -from collections import Counter - - -def gen_file_metadata(blob): - """ - Takes a file blob (bytestream) and returns hashes and other metadata. - - Returns a dict: size_bytes, md5hex, sha1hex, sha256hex, mimetype - """ - assert blob - mimetype = magic.Magic(mime=True).from_buffer(blob) - hashes = [ - hashlib.sha1(), - hashlib.sha256(), - hashlib.md5(), - ] - for h in hashes: - h.update(blob) - return dict( - size_bytes=len(blob), - sha1hex=hashes[0].hexdigest(), - sha256hex=hashes[1].hexdigest(), - md5hex=hashes[2].hexdigest(), - mimetype=mimetype, - ) - -def requests_retry_session(retries=2, backoff_factor=3, - status_forcelist=(500, 502, 504), session=None): - """ - From: https://www.peterbe.com/plog/best-practice-with-retries-with-requests - """ - session = session or requests.Session() - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=backoff_factor, - status_forcelist=status_forcelist, - ) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - return session - - -class DeliverFatcatDisk: - - def __init__(self, disk_dir, **kwargs): - self.count = Counter() - self.disk_dir = disk_dir - self.disk_prefix = kwargs.get('disk_prefix', 'pdf/') - self.disk_suffix = kwargs.get('disk_suffix', '.pdf') - self.session = requests_retry_session() - self.session.headers.update({ - 'User-Agent': 'Mozilla/5.0 fatcat.DeliverFatcatDisk', - }) - - def run(self, release_json_file): - sys.stderr.write("Ensuring all 256 base directories exist...\n") - for i in range(256): - fpath = "{}/{}{:02x}".format( - self.disk_dir, - self.disk_prefix, - i) - os.makedirs(fpath, exist_ok=True) - sys.stderr.write("Starting...\n") - for line in release_json_file: - self.count['total'] += 1 - if not line.startswith('{'): - self.count['skip-no-release'] += 1 - continue - #print(line) - release = json.loads(line) - assert 'ident' in release - self.fetch_release(release) - sys.stderr.write("{}\n".format(self.count)) - - def blob_path(self, sha1hex): - fpath = "{}/{}{}/{}{}".format( - self.disk_dir, - self.disk_prefix, - sha1hex[0:2], - sha1hex, - self.disk_suffix) - return fpath - - def does_file_already_exist(self, sha1hex): - return os.path.isfile(self.blob_path(sha1hex)) - - def filter_files(self, files): - """ - Takes a list of file entities and only returns the ones which are PDFs - we can download. - """ - good = [] - for f in files: - if f['mimetype'] and not 'pdf' in f['mimetype'].lower(): - continue - for url in f['urls']: - if 'archive.org/' in url['url']: - good.append(f) - break - return good - - def fetch_content(self, url): - """ - Returns tuple: (str:status, content) - Content contains bytes only if status is "success", otherwise None - """ - if '://web.archive.org/' in url: - # add id_ to URL to avoid wayback re-writing - l = url.split('/') - if l[2] == 'web.archive.org' and l[3] == 'web' and not '_' in l[4]: - l[4] = l[4] + 'id_' - url = '/'.join(l) - - try: - resp = self.session.get(url) - except requests.exceptions.RetryError: - return ('wayback-error', None) - except requests.exceptions.TooManyRedirects: - return ('too-many-redirects', None) - if resp.status_code != 200: - return ('fetch:{}'.format(resp.status_code), None) - else: - return ('success', resp.content) - - def fetch_file(self, f): - """ - Returns tuple: (status, sha1hex, file_meta) - - file_meta is a dict on success, or None otherwise - """ - sha1hex = f['sha1'] - if self.does_file_already_exist(sha1hex): - return ('exists', sha1hex, None) - status = None - for url in f['urls']: - url = url['url'] - if not 'archive.org' in url: - continue - status, content = self.fetch_content(url) - if status == 'success': - # TODO: verify sha1hex - file_meta = gen_file_metadata(content) - if file_meta['sha1hex'] != sha1hex: - status = 'sha1-mismatch' - continue - with open(self.blob_path(sha1hex), 'wb') as outf: - outf.write(content) - return ('success', sha1hex, file_meta) - if status: - return (status, sha1hex, None) - else: - return ('no-urls', sha1hex, None) - - def fetch_release(self, release): - good_files = self.filter_files(release['files']) - status = 'no-file' - sha1hex = None - for f in good_files: - status, sha1hex, file_meta = self.fetch_file(f) - if status in ('success', 'exists'): - break - else: - continue - if sha1hex: - print("{}\t{}".format(status, sha1hex)) - else: - print(status) - self.count[status] += 1 - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument('--disk-dir', - required=True, - type=str, - help='local base directory to save into') - parser.add_argument('--disk-prefix', - type=str, - default="pdf/", - help='directory prefix for items created in bucket') - parser.add_argument('--disk-suffix', - type=str, - default=".pdf", - help='file suffix for created files') - parser.add_argument('release_json_file', - help="JSON manifest of fatcat release entities", - default=sys.stdin, - type=argparse.FileType('r')) - args = parser.parse_args() - - worker = DeliverFatcatDisk(**args.__dict__) - worker.run(args.release_json_file) - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/scripts/parse_cord19_csv.py b/scripts/parse_cord19_csv.py deleted file mode 100755 index 536e5d3..0000000 --- a/scripts/parse_cord19_csv.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import csv -import json - -CSVFILE = sys.argv[1] - -with open(CSVFILE, newline='') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - row = dict(row) - row['mag_id'] = row.pop('Microsoft Academic Paper ID') - row['who_covidence_id'] = row.pop('WHO #Covidence').replace('#', '') - print(json.dumps(row, sort_keys=True)) |