aboutsummaryrefslogtreecommitdiffstats
path: root/python/scripts/covid2ingestrequest.py
blob: 33c425d25812d9665391fe6faeb0dcb8d6fadbe0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/usr/bin/env python3

"""
Transform an unpaywall dump (JSON) into ingest requests.
"""

import sys
import json
import argparse
import urlcanon


def canon(s):
    parsed = urlcanon.parse_url(s)
    return str(urlcanon.whatwg(parsed))


def transform_cnki(obj):

    requests = []
    assert obj['cnki_id']


    requests = []
    requests.append({
        'base_url': canon(obj['info_url']),
        'ingest_type': 'pdf',
        'link_source': 'cnki_covid19',
        'link_source_id': obj['cnki_id'],
        'ingest_request_source': 'scrape-covid19',
    })
    if 'read_url' in obj:
        requests.append({
            'base_url': canon(obj['read_url']),
            'ingest_type': 'pdf',  # actually HTML
            'link_source': 'cnki_covid19',
            'link_source_id': obj['cnki_id'],
            'ingest_request_source': 'scrape-covid19',
        })

    return requests

def transform_wanfang(obj):

    assert obj['wanfang_id']
    return [{
        'base_url': canon(obj['url']),
        'ingest_type': 'pdf',
        'link_source': 'wanfang_covid19',
        'link_source_id': obj['wanfang_id'],
        'ingest_request_source': 'scrape-covid19',
    }]


def run(args):
    for l in args.json_file:
        if not l.strip():
            continue
        row = json.loads(l)

        if 'wanfang_id' in row:
            requests = transform_wanfang(row) or []
        elif 'cnki_id' in row:
            requests = transform_cnki(row) or []
        else:
            continue
        for r in requests:
            print("{}".format(json.dumps(r, sort_keys=True)))

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('json_file',
        help="COVID-19 metadata file to use",
        type=argparse.FileType('r'))
    subparsers = parser.add_subparsers()

    args = parser.parse_args()

    run(args)

if __name__ == '__main__':
    main()