aboutsummaryrefslogtreecommitdiffstats
path: root/notes/cleanups/scripts/file2ingestrequest.py
blob: a005837f432a8633ca7237d3cdd034eb3634b820 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#!/usr/bin/env python3

from typing import Optional
import json, sys


def transform(row: dict) -> Optional[dict]:
    if row.get('mimetype') not in [None, 'application/pdf']:
        return None
    if row.get('state') != 'active':
        return None
    base_url = None
    for url in (row.get('urls') or []):
        url = url['url']
        if '://web.archive.org/' not in url and '://archive.org/' not in url:
            base_url = url
            break
    if not base_url:
        return None
    if not row.get('sha1'):
        return None
    return dict(
        base_url=base_url,
        ingest_type="pdf",
        link_source="fatcat",
        link_source_id=f"file_{row['ident']}",
        ingest_request_source="file-backfill",
        ext_ids=dict(
            sha1=row['sha1'],
        ),
    )


def run():
    for l in sys.stdin:
        if not l.strip():
            continue
        row = json.loads(l)
        request = transform(row)
        if request:
            print(json.dumps(request, sort_keys=True))

if __name__=="__main__":
    run()