diff options
| -rw-r--r-- | python/fatcat/api.py | 12 | ||||
| -rw-r--r-- | python/fatcat/api_client.py | 88 | ||||
| -rw-r--r-- | python/tests/fixtures.py | 12 |
3 files changed, 63 insertions, 49 deletions
diff --git a/python/fatcat/api.py b/python/fatcat/api.py index 2c91533b..1228095b 100644 --- a/python/fatcat/api.py +++ b/python/fatcat/api.py @@ -36,7 +36,7 @@ def api_work_create(params=None): # TODO: Special-case to pull out primary and create that? if params == None: params = request.get_json() - editgroup = get_or_create_editgroup(params.get('editgroup')) + editgroup = get_or_create_editgroup(params.get('editgroup_id')) rev = WorkRev( title=params.get('title', None), work_type=params.get('work_type', None), @@ -64,7 +64,7 @@ def api_release_get(ident): def api_release_create(params=None): if params == None: params = request.get_json() - editgroup = get_or_create_editgroup(params.get('editgroup')) + editgroup = get_or_create_editgroup(params.get('editgroup_id')) creators = params.get('creators', []) creators = [CreatorIdent.query.get_or_404(c) for c in creators] targets = [ref['target'] for ref in params.get('refs', []) if ref.get('target') != None] @@ -132,7 +132,7 @@ def api_creator_get(ident): def api_creator_create(params=None): if params == None: params = request.get_json() - editgroup = get_or_create_editgroup(params.get('editgroup')) + editgroup = get_or_create_editgroup(params.get('editgroup_id')) rev = CreatorRev( name=params.get('name', None), orcid=params.get('orcid', None), @@ -168,7 +168,7 @@ def api_container_get(ident): def api_container_create(params=None): if params == None: params = request.get_json() - editgroup = get_or_create_editgroup(params.get('editgroup')) + editgroup = get_or_create_editgroup(params.get('editgroup_id')) rev = ContainerRev( name=params.get('name', None), publisher=params.get('publisher', None), @@ -184,7 +184,7 @@ def api_container_create(params=None): @app.route('/v0/container/lookup', methods=['GET']) def api_container_lookup(): - params = request.get_json() + params = request.args issn = params['issn'].strip() # TODO: proper regex if not (len(issn) == 9 and issn[0:4].isdigit() and issn[5:7].isdigit()): @@ -205,7 +205,7 @@ def api_file_get(ident): def api_file_create(params=None): if params == None: params = request.get_json() - editgroup = get_or_create_editgroup(params.get('editgroup')) + editgroup = get_or_create_editgroup(params.get('editgroup_id')) releases = params.get('releases', []) releases = [ReleaseIdent.query.get_or_404(r) for r in releases] rev = FileRev( diff --git a/python/fatcat/api_client.py b/python/fatcat/api_client.py index f2fd6a1d..6497e977 100644 --- a/python/fatcat/api_client.py +++ b/python/fatcat/api_client.py @@ -12,19 +12,28 @@ class FatCatApiClient: self._issn_map = dict() def get(self, path, data=None): - headers = {"content-type": "application/json"} - return self.session.get(self.host_url + path, json=data, + headers = {"Content-Type": "application/json"} + rv = self.session.get(self.host_url + path, params=data, headers=headers) + #if rv.status_code not in [200, 201]: + # print(rv) + # print(rv.content) + return rv def post(self, path, data=None): - headers = {"content-type": "application/json"} - return self.session.post(self.host_url + path, json=data, + headers = {"Content-Type": "application/json"} + rv = self.session.post(self.host_url + path, json=data, headers=headers) + if rv.status_code not in [200, 201]: + #print(rv) + #print(rv.content) + pass + return rv def new_editgroup(self): rv = self.post('/v0/editgroup', data=dict( - editor=1)) - assert rv.status_code == 200 + editor_id=1)) + assert rv.status_code in [200, 201] editgroup_id = rv.json()['id'] return editgroup_id @@ -39,8 +48,8 @@ class FatCatApiClient: return self._issn_map[issn] rv = self.get('/v0/container/lookup', data=dict(issn=issn)) container_id = None - if rv.status_code == 200: - container_id = rv.json()['id'] + if rv.status_code in [200, 201]: + container_id = rv.json().get('ident', rv.json().get('id')) else: # only other valid response is a 404; otherwise we had an error assert rv.status_code == 404 @@ -50,24 +59,28 @@ class FatCatApiClient: def import_crossref_file(self, json_file, create_containers=False, batchsize=100): eg = self.new_editgroup() i = 0 - with open(json_file, 'r') as file: - for line in file: - if i % batchsize == 0: - sys.stdout.write('\n{}: '.format(i)) - if (i+1) % 20 == 0: - sys.stdout.write('.') - i = i + 1 - obj = json.loads(line) - if not ("author" in obj and "title" in obj): - continue - try: - self.import_crossref_dict(obj, editgroup=eg, - create_containers=create_containers) - except Exception as e: - print("ERROR: {}".format(e)) - if i % batchsize == 0: - self.accept_editgroup(eg) - eg = self.new_editgroup() + if json_file == "-": + stream = sys.stdin + else: + stream = open(json_file, 'r') + for line in stream: + if i % batchsize == 0: + sys.stdout.write('\n{}: '.format(i)) + if (i+1) % 20 == 0: + sys.stdout.write('.') + i = i + 1 + obj = json.loads(line) + if not ("author" in obj and "title" in obj): + continue + try: + self.import_crossref_dict(obj, editgroup=eg, + create_containers=create_containers) + except Exception as e: + print("CROSSREF IMPORT ERROR: {}".format(e)) + #raise e + if i % batchsize == 0: + self.accept_editgroup(eg) + eg = self.new_editgroup() if i % batchsize != 0: self.accept_editgroup(eg) print("done!") @@ -89,7 +102,7 @@ class FatCatApiClient: container = dict( issn=issn, name=meta['container-title'][0], - container=container_id, + container_id=container_id, #sortname=meta['short-container-title'][0]) publisher=meta['publisher']) @@ -97,10 +110,10 @@ class FatCatApiClient: rv = self.post('/v0/container', data=dict( issn=container['issn'], publisher=container['publisher'])) - assert rv.status_code == 200 - container_id = rv.json()['id'] + assert rv.status_code in [200, 201] + container_id = rv.json().get('ident', rv.json().get('id')) print("created container: {}".format(issn)) - container['id'] = container_id + container['ident'] = container_id self._issn_map[issn] = container_id # references @@ -116,9 +129,10 @@ class FatCatApiClient: # work and release title = meta['title'][0] rv = self.post('/v0/work', - data=dict(title=title, editgroup=editgroup)) #work_type="book" - assert rv.status_code == 200 - work_id = rv.json()['id'] + data=dict(title=title, editgroup_id=editgroup)) #work_type="book" + assert rv.status_code in [200, 201] + #print(rv.json()) + work_id = rv.json().get('ident', rv.json().get('id')) extra = dict(crossref={ 'links': meta.get('link', []), @@ -128,7 +142,7 @@ class FatCatApiClient: rv = self.post('/v0/release', data=dict( title=title, - work=work_id, + work_id=work_id, # XXX: creators=creators, # XXX: refs=refs, # XXX: container=container_id, @@ -139,10 +153,10 @@ class FatCatApiClient: issue=meta.get('issue', None), volume=meta.get('volume', None), pages=meta.get('page', None), - editgroup=editgroup, + editgroup_id=editgroup, extra=extra)) - assert rv.status_code == 200 - release_id = rv.json()['id'] + assert rv.status_code in [200, 201] + release_id = rv.json().get('ident', rv.json().get('id')) def import_issn_file(self, json_file, create_containers=False, batchsize=100): eg = self.new_editgroup() diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py index d3d8c24b..364c8be7 100644 --- a/python/tests/fixtures.py +++ b/python/tests/fixtures.py @@ -61,7 +61,7 @@ def enrichen_test_app(app): name="schmournal", publisher="society of authors", issn="2222-3333", - editgroup=editgroup_id, + editgroup_id=editgroup_id, extra=dict(a=2, i="zing"))), headers={"content-type": "application/json"}) assert rv.status_code == 200 @@ -72,7 +72,7 @@ def enrichen_test_app(app): data=json.dumps(dict( name="anon y. mouse", orcid="0000-0002-1825-0097", - editgroup=editgroup_id, + editgroup_id=editgroup_id, extra=dict(w=1, q="zing"))), headers={"content-type": "application/json"}) assert rv.status_code == 200 @@ -83,7 +83,7 @@ def enrichen_test_app(app): data=json.dumps(dict( title="dummy work", work_type="book", - editgroup=editgroup_id, + editgroup_id=editgroup_id, extra=dict(a=3, b="zing"))), headers={"content-type": "application/json"}) assert rv.status_code == 200 @@ -98,7 +98,7 @@ def enrichen_test_app(app): work=work_id, creators=[creator_id], doi="10.1234/58", - editgroup=editgroup_id, + editgroup_id=editgroup_id, refs=[ dict(stub="some other journal article"), ], @@ -116,7 +116,7 @@ def enrichen_test_app(app): container=container_id, creators=[creator_id], doi="10.1234/5678", - editgroup=editgroup_id, + editgroup_id=editgroup_id, refs=[ dict(stub="some book", target=stub_release_id), ], @@ -131,7 +131,7 @@ def enrichen_test_app(app): sha1="deadbeefdeadbeef", size=1234, releases=[release_id], - editgroup=editgroup_id, + editgroup_id=editgroup_id, extra=dict(f=4, b="zing"))), headers={"content-type": "application/json"}) assert rv.status_code == 200 |