aboutsummaryrefslogtreecommitdiffstats
path: root/fuzzycat
diff options
context:
space:
mode:
authorMartin Czygan <martin.czygan@gmail.com>2020-12-09 14:04:46 +0100
committerMartin Czygan <martin.czygan@gmail.com>2020-12-09 14:04:46 +0100
commitb1118bb87efef2d4a5f252ced53f1c59246fec8e (patch)
tree5f6a4837df42b3c0e66087f3fd0954e81f303e12 /fuzzycat
parent3c7378e89687846fe7701212ba90194686990695 (diff)
downloadfuzzycat-b1118bb87efef2d4a5f252ced53f1c59246fec8e.tar.gz
fuzzycat-b1118bb87efef2d4a5f252ced53f1c59246fec8e.zip
add two more cases
Diffstat (limited to 'fuzzycat')
-rw-r--r--fuzzycat/common.py4
-rw-r--r--fuzzycat/verify.py25
2 files changed, 27 insertions, 2 deletions
diff --git a/fuzzycat/common.py b/fuzzycat/common.py
index 977db2c..c50962b 100644
--- a/fuzzycat/common.py
+++ b/fuzzycat/common.py
@@ -24,6 +24,7 @@ class OK(str, Enum):
DOI = 'ok.doi'
DUMMY = 'ok.dummy'
FIGSHARE_VERSION = 'ok.figshare_version'
+ JACCARD_AUTHORS = 'ok.jaccard_authors'
PMID_DOI_PAIR = 'ok.pmid_doi_pair'
PREPRINT_PUBLISHED = 'ok.preprint_published'
SLUG_TITLE_AUTHOR_MATCH = 'ok.slug_title_author_match'
@@ -51,10 +52,11 @@ class Miss(str, Enum):
CUSTOM_PREFIX_10_7916 = 'miss.custom_prefix_10_7916'
CUSTOM_VHS = 'miss.vhs' # https://fatcat.wiki/release/44gk5ben5vghljq6twm7lwmxla
DATASET_DOI = 'miss.dataset_doi'
+ JSTOR_ID = 'miss.jstor_id'
NUM_DIFF = 'miss.num_diff'
RELEASE_TYPE = 'miss.release_type'
+ SHARED_DOI_PREFIX = 'miss.shared_doi_prefix'
SHORT_TITLE = 'miss.short_title'
SUBTITLE = 'miss.subtitle'
TITLE_FILENAME = 'miss.title_filename'
YEAR = 'miss.year'
- SHARED_DOI_PREFIX = 'miss.shared_doi_prefix'
diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py
index dd2ced8..9b6d217 100644
--- a/fuzzycat/verify.py
+++ b/fuzzycat/verify.py
@@ -76,7 +76,7 @@ from glom import PathAccessError, glom
from fuzzycat.common import OK, Miss, Status
from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, has_doi_prefix,
- num_project, slugify_string)
+ num_project, slugify_string, jaccard)
# The result of clustering are documents that have a key k and a list of values
# (of the cluster) v.
@@ -437,6 +437,15 @@ def compare(a, b):
(dict_key_exists(b, "ext_ids.pmid") and not dict_key_exists(b, "ext_ids.doi"))):
return (Status.STRONG, OK.PMID_DOI_PAIR)
+ # Two JSTOR items will probably be different.
+ try:
+ a_jstor_id = glom(a, "ext_ids.jstor")
+ b_jstor_id = glom(b, "ext_ids.jstor")
+ if a_jstor_id != b_jstor_id:
+ return (Status.DIFFERENT, Miss.JSTOR_ID)
+ except PathAccessError:
+ pass
+
# Publication from same publisher and different DOI or year a probably
# different.
try:
@@ -473,6 +482,20 @@ def compare(a, b):
avg_score = sum(top_scores) / len(top_scores)
if avg_score > 0.5:
return (Status.STRONG, OK.TOKENIZED_AUTHORS)
+ else:
+ # Kuidong Xu, Joong Ki Choi, Eun Jin Yang, Kyu Chul Lee, Yanli Lei
+ # J.K. Choi, K. Xu, E.J. Yang, K.C. Lee, Y. Lei
+ # 0.2942857142857143
+ print("author comp score: {}".format(avg_score))
+
+ # Fallback jaccard token comparison.
+ # Kuidong Xu, Joong Ki Choi, Eun Jin Yang, Kyu Chul Lee, Yanli Lei
+ # J.K. Choi, K. Xu, E.J. Yang, K.C. Lee, Y. Lei
+ # avg_score was 0.2942857142857143, but jaccard ~0.38
+ a_tok = [tok for tok in re.findall(r"[\w]{3,}", " ".join(a_slug_authors)) if tok]
+ b_tok = [tok for tok in re.findall(r"[\w]{3,}", " ".join(b_slug_authors)) if tok]
+ if jaccard(set(a_tok), set(b_tok)) > 0.35:
+ return (Status.STRONG, OK.JACCARD_AUTHORS)
# TODO: This misses spelling differences, e.g.
# https://fatcat.wiki/release/7nbcgsohrrak5cuyk6dnit6ega and