aboutsummaryrefslogtreecommitdiffstats
path: root/python/ia_pdf_match.py
blob: 60b7843bbe89bf2e5e6a6272ca5e29a23ceffcc9 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/usr/bin/env python3

"""
Input is IA item metadata JSON.
Ouput is insertable fatcat "match" JSON

- md5
- sha1
- sha256
- size
- urls
- cdx (list; empty here)

- dois (list)
- pmcid
- jstor_id
- arxiv_id

When invoking import matched, be sure to:

    --default-link-rel repository (?)
    --default-mimetype application/pdf
"""

import sys
import json

def parse(obj):
    if obj['metadata']['identifier'].endswith('-test') or obj['metadata'].get('test'):
        sys.stderr.write('skip: test item\n')
        return None

    extid_type = None
    extid = None
    if obj['metadata']['identifier'].startswith('arxiv-'):
        extid_type = 'arxiv_id'
        extid = obj['metadata'].get('source')
        if not extid:
            sys.stderr.write('skip: no source\n')
            return None
        assert extid.startswith('http://arxiv.org/abs/')
        extid = extid.replace('http://arxiv.org/abs/', '')
        #print(extid)
        assert '/' in extid or '.' in extid
        if not 'v' in extid or not extid[-1].isdigit():
            sys.stderr.write('skip: non-versioned arxiv_id\n')
            return None
    elif obj['metadata']['identifier'].startswith('paper-doi-10_'):
        extid_type = 'doi'
        extid = obj['metadata']['identifier-doi']
        assert extid.startswith("10.")
    elif obj['metadata']['identifier'].startswith('pubmed-PMC'):
        extid_type = 'pmcid'
        extid = obj['metadata']['identifier'].replace('pubmed-', '')
        assert extid.startswith("PMC")
        int(extid[3:])
    elif obj['metadata']['identifier'].startswith('jstor-'):
        extid_type = 'jstor_id'
        extid = obj['metadata']['identifier'].replace('jstor-', '')
        int(extid)
    else:
        raise NotImplementedError()

    pdf_file = None
    for f in obj['files']:
        if f['source'] == "original" and "PDF" in f['format']:
            pdf_file = f
            break
    if not pdf_file:
        sys.stderr.write('skip: no PDF found: {}\n'.format(obj['metadata']['identifier']))
        #for f in obj['files']:
        #    sys.stderr.write(f['format'] + "\n")
        return None

    assert pdf_file['name'].endswith('.pdf')

    match = {
        'md5': pdf_file['md5'],
        'sha1': pdf_file['sha1'],
        'size': int(pdf_file['size']),
        'mimetype': 'application/pdf',
        'urls': [
            "https://archive.org/download/{}/{}".format(
                obj['metadata']['identifier'],
                pdf_file['name']),
        ],
        'cdx': [],
        'dois': [],
    }

    if extid_type == 'doi':
        match['dois'] = [extid,]
    else:
        match[extid_type] = extid

    return match

def run():
    for line in sys.stdin:
        if not line:
            continue
        obj = json.loads(line)
        match = parse(obj)
        if match:
            print(json.dumps(match))

if __name__ == '__main__':
    run()