1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
# coding: utf-8
"""
Public API for fuzzy matches for fatcat.
Match methods return candidates, verify methods return a match status.
match_containar_fuzzy -> List[ContainerEntity]
match_release_fuzzy -> List[ReleaseEntity]
verify_serial_name -> MatchStatus
verify_container_name -> MatchStatus
verify_container_fuzzy -> MatchStatus
verify_release_fuzzy -> MatchStatus
Candidate generation will use external data from search and hence is expensive. Verification is fast.
"""
from typing import List, Optional, Union, Set
import elasticsearch
from fatcat_openapi_client import (ApiException, ContainerEntity, DefaultApi, ReleaseEntity,
ReleaseExtIds, WorkEntity)
from fatcat_openapi_client.api.default_api import DefaultApi
from fuzzycat.fatcat.common import MatchStatus, response_to_entity_list
from fuzzycat.serials import serialsdb
from fuzzycat import cleanups
def match_container_fuzzy(container: ContainerEntity,
size: int = 5,
es: Optional[Union[str, elasticsearch.client.Elasticsearch]] = None,
api: Optional[DefaultApi] = None) -> List[ContainerEntity]:
"""
Given a container entity, which can be (very) partial, return a list of
candidate matches. Elasticsearch can be a hostport or the low level client
object.
Random data point: with 20 parallel workers callind match_container_fuzzy,
we get around 40 req/s.
"""
assert isinstance(container, ContainerEntity)
if size is None or size == 0:
size = 10000 # or any large number
if isinstance(es, str):
es = elasticsearch.Elasticsearch([es])
if es is None:
es = elasticsearch.Elasticsearch()
# If we find any match by ISSN-L, we return only those.
if container.issnl:
s = (elasticsearch_dsl.Search(using=es, index="fatcat_container").query(
"term", issns=container.issnl).extra(size=size))
resp = s.execute()
if len(resp) > 0:
return response_to_entity_list(resp, entity_type=ContainerEntity, api=api)
# Do we have an exact QID match?
if container.wikidata_qid:
s = (elasticsearch_dsl.Search(using=es, index="fatcat_container").query(
"term", wikidata_qid=container.wikidata_qid).extra(size=size))
resp = s.execute()
if len(resp) > 0:
return response_to_entity_list(resp, entity_type=ContainerEntity, api=api)
# Start with exact name match.
#
# curl -s https://search.fatcat.wiki/fatcat_container/_mapping | jq .
#
# "name": {
# "type": "text",
# "copy_to": [
# "biblio"
# ],
# "analyzer": "textIcu",
# "search_analyzer": "textIcuSearch"
# },
#
body = {
"query": {
"match": {
"name": {
"query": container.name,
"operator": "AND"
}
}
},
"size": size,
}
resp = es.search(body=body, index="fatcat_container")
if resp["hits"]["total"] > 0:
return response_to_entity_list(resp, entity_type=ContainerEntity, api=api)
# Get fuzzy.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness
body = {
"query": {
"match": {
"name": {
"query": container.name,
"operator": "AND",
"fuzziness": "AUTO",
}
}
},
"size": size,
}
resp = es.search(body=body, index="fatcat_container")
if resp["hits"]["total"] > 0:
return response_to_entity_list(resp, entity_type=ContainerEntity, api=api)
return []
def match_release_fuzzy(release: ReleaseEntity,
size: int = 5,
es: Optional[Union[str, elasticsearch.client.Elasticsearch]] = None,
api: Optional[DefaultApi] = None) -> List[ReleaseEntity]:
"""
Given a release entity, return a number similar release entities from
fatcat using Elasticsearch.
"""
assert isinstance(release, ReleaseEntity)
if size is None or size == 0:
size = 10000 # or any large number
if isinstance(es, str):
es = elasticsearch.Elasticsearch([es])
if es is None:
es = elasticsearch.Elasticsearch()
# Try to match by external identifier.
ext_ids = release.ext_ids
attrs = {
"doi": "doi",
"wikidata_qid": "wikidata_qid",
"isbn13": "isbn13",
"pmid": "pmid",
"pmcid": "pmcid",
"core": "code_id",
"arxiv": "arxiv_id",
"jstor": "jstor_id",
"ark": "ark_id",
"mag": "mag_id",
}
for attr, es_field in attrs.items():
value = getattr(ext_ids, attr)
if not value:
continue
s = (elasticsearch_dsl.Search(using=es,
index="fatcat_release").query("term", **{
es_field: value
}).extra(size=size))
resp = s.execute()
if len(resp) > 0:
return response_to_entity_list(resp, entity_type=ReleaseEntity, api=api)
body = {
"query": {
"match": {
"title": {
"query": release.title,
"operator": "AND"
}
}
},
"size": size,
}
resp = es.search(body=body, index="fatcat_release")
if resp["hits"]["total"] > 0:
return response_to_entity_list(resp, entity_type=ReleaseEntity, api=api)
# Get fuzzy.
# https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness
body = {
"query": {
"match": {
"title": {
"query": release.title,
"operator": "AND",
"fuzziness": "AUTO",
}
}
},
"size": size,
}
resp = es.search(body=body, index="fatcat_release")
if resp["hits"]["total"] > 0:
return response_to_entity_list(resp, entity_type=ReleaseEntity, api=api)
return []
def verify_serial_name(a: str, b: str) -> MatchStatus:
"""
Serial name verification. Serial names are a subset of container names.
There are about 2M serials.
"""
def verify(a : Set[str], b : Set[str]) -> MatchStatus:
# If any name yields multiple ISSN-L, we cannot decide.
if len(a) > 1:
return MatchStatus.AMBIGIOUS
if len(b) > 1:
return MatchStatus.AMBIGIOUS
# If both names point the same ISSN-L, it is an exact match.
if len(a) > 0 and len(a) == len(b):
if len(a & b) == len(a):
return MatchStatus.EXACT
else:
return MatchStatus.DIFFERENT
# Multiple names possible, but there is overlap.
if len(a & b) > 0:
return MatchStatus.STRONG
# First, try values as given.
issnls_for_a = serialsdb.get(a, set())
issnls_for_b = serialsdb.get(b, set())
status = verify(issnls_for_a, issnls_for_b)
if status != MatchStatus.AMBIGIOUS:
return status
# Try to match slightly cleaned up values.
issnls_for_a = serialsdb.get(a, set(), cleanup_pipeline=cleanups.basic)
issnls_for_b = serialsdb.get(b, set(), cleanup_pipeline=cleanups.basic)
return verify(issnls_for_a, issnls_for_b)
def verify_container_name(a: str, b: str) -> MatchStatus:
status = verify_serial_name(a, b)
if status != MatchStatus.AMBIGIOUS:
return status
# TODO: add additional verification, string match and common patterns.
def verify_container_match(a: ContainerEntity, b: ContainerEntity) -> MatchStatus:
pass
def verify_release_match(a: ReleaseEntity, b: ReleaseEntity) -> MatchStatus:
pass
|
