1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
import sys
import json
import itertools
import fatcat_client
def value_or_none(e):
if type(e) == dict:
e = e.get('value')
if type(e) == str and len(e) == 0:
e = None
return e
# from: https://docs.python.org/3/library/itertools.html
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
args = [iter(iterable)] * n
return itertools.zip_longest(*args, fillvalue=fillvalue)
class FatcatOrcidImporter:
def __init__(self, host_url):
conf = fatcat_client.Configuration()
conf.host = host_url
self.api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf))
def parse_orcid_dict(self, obj):
"""
obj is a python dict (parsed from json).
returns a CreatorEntity
"""
name = obj['person']['name']
if name is None:
return None
extra = None
given = value_or_none(name.get('given-name'))
sur = value_or_none(name.get('family-name'))
display = value_or_none(name.get('credit-name'))
if display is None:
# TODO: sorry human beings
display = "{} {}".format(given, sur)
ce = fatcat_client.CreatorEntity(
orcid=obj['orcid-identifier']['path'],
given_name=given,
surname=sur,
display_name=display,
extra=extra)
return ce
def process_line(self, line, editgroup_id=None):
"""Doesn't accept the editgroup"""
obj = json.loads(line)
ce = self.parse_orcid_dict(obj)
if ce is not None:
ce.editgroup_id = editgroup_id
self.api.create_creator(ce)
def process_source(self, source, group_size=100):
"""Creates and auto-accepts editgropu every group_size lines"""
eg = self.api.create_editgroup(fatcat_client.Editgroup(editor_id=1))
for i, line in enumerate(source):
self.process_line(line, editgroup_id=eg.id)
if i > 0 and (i % group_size) == 0:
self.api.accept_editgroup(eg)
eg = self.api.create_editgroup(fatcat_client.Editgroup(editor_id=1))
if i == 0 or (i % group_size) != 0:
self.api.accept_editgroup(eg.id)
def process_batch(self, source, size=50):
"""Reads and processes in batches (not API-call-per-line)"""
for lines in grouper(source, size):
objects = [self.parse_orcid_dict(json.loads(l))
for l in lines if l != None]
objects = [o for o in objects if o != None]
eg = self.api.create_editgroup(fatcat_client.Editgroup(editor_id=1))
for o in objects:
o.editgroup_id = eg.id
self.api.create_creator_batch(objects)
self.api.accept_editgroup(eg.id)
print("inserted {}".format(len(objects)))
|
