add two more cases

9 files changed, 208 insertions, 3 deletions

diff --git a/fuzzycat/common.py b/fuzzycat/common.py
index 977db2c..c50962b 100644
--- a/fuzzycat/common.py
+++ b/fuzzycat/common.py
@@ -24,6 +24,7 @@ class OK(str, Enum):
     DOI = 'ok.doi'
     DUMMY = 'ok.dummy'
     FIGSHARE_VERSION = 'ok.figshare_version'
+    JACCARD_AUTHORS = 'ok.jaccard_authors'
     PMID_DOI_PAIR = 'ok.pmid_doi_pair'
     PREPRINT_PUBLISHED = 'ok.preprint_published'
     SLUG_TITLE_AUTHOR_MATCH = 'ok.slug_title_author_match'
@@ -51,10 +52,11 @@ class Miss(str, Enum):
     CUSTOM_PREFIX_10_7916 = 'miss.custom_prefix_10_7916'
     CUSTOM_VHS = 'miss.vhs'  # https://fatcat.wiki/release/44gk5ben5vghljq6twm7lwmxla
     DATASET_DOI = 'miss.dataset_doi'
+    JSTOR_ID = 'miss.jstor_id'
     NUM_DIFF = 'miss.num_diff'
     RELEASE_TYPE = 'miss.release_type'
+    SHARED_DOI_PREFIX = 'miss.shared_doi_prefix'
     SHORT_TITLE = 'miss.short_title'
     SUBTITLE = 'miss.subtitle'
     TITLE_FILENAME = 'miss.title_filename'
     YEAR = 'miss.year'
-    SHARED_DOI_PREFIX = 'miss.shared_doi_prefix'
diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py
index dd2ced8..9b6d217 100644
--- a/fuzzycat/verify.py
+++ b/fuzzycat/verify.py
@@ -76,7 +76,7 @@ from glom import PathAccessError, glom
 
 from fuzzycat.common import OK, Miss, Status
 from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, has_doi_prefix,
-                            num_project, slugify_string)
+                            num_project, slugify_string, jaccard)
 
 # The result of clustering are documents that have a key k and a list of values
 # (of the cluster) v.
@@ -437,6 +437,15 @@ def compare(a, b):
                 (dict_key_exists(b, "ext_ids.pmid") and not dict_key_exists(b, "ext_ids.doi"))):
                 return (Status.STRONG, OK.PMID_DOI_PAIR)
 
+    # Two JSTOR items will probably be different.
+    try:
+        a_jstor_id = glom(a, "ext_ids.jstor")
+        b_jstor_id = glom(b, "ext_ids.jstor")
+        if a_jstor_id != b_jstor_id:
+            return (Status.DIFFERENT, Miss.JSTOR_ID)
+    except PathAccessError:
+        pass
+
     # Publication from same publisher and different DOI or year a probably
     # different.
     try:
@@ -473,6 +482,20 @@ def compare(a, b):
             avg_score = sum(top_scores) / len(top_scores)
             if avg_score > 0.5:
                 return (Status.STRONG, OK.TOKENIZED_AUTHORS)
+            else:
+                # Kuidong Xu, Joong Ki Choi, Eun Jin Yang, Kyu Chul Lee, Yanli Lei
+                # J.K. Choi, K. Xu, E.J. Yang, K.C. Lee, Y. Lei
+                # 0.2942857142857143
+                print("author comp score: {}".format(avg_score))
+
+        # Fallback jaccard token comparison.
+        # Kuidong Xu, Joong Ki Choi, Eun Jin Yang, Kyu Chul Lee, Yanli Lei
+        # J.K. Choi, K. Xu, E.J. Yang, K.C. Lee, Y. Lei
+        # avg_score was 0.2942857142857143, but jaccard ~0.38
+        a_tok = [tok for tok in re.findall(r"[\w]{3,}", " ".join(a_slug_authors)) if tok]
+        b_tok = [tok for tok in re.findall(r"[\w]{3,}", " ".join(b_slug_authors)) if tok]
+        if jaccard(set(a_tok), set(b_tok)) > 0.35:
+            return (Status.STRONG, OK.JACCARD_AUTHORS)
 
         # TODO: This misses spelling differences, e.g.
         # https://fatcat.wiki/release/7nbcgsohrrak5cuyk6dnit6ega and
diff --git a/notes/2020_11_testruns.md b/notes/2020_11_testruns.md
index 5e974dd..9fb1508 100644
--- a/notes/2020_11_testruns.md
+++ b/notes/2020_11_testruns.md
@@ -148,6 +148,9 @@ Seems wo work.
 > choice review
 
 * [ ] https://fatcat.wiki/release/b7bbygyawzdsthai7j7rmztrxe https://fatcat.wiki/release/mvvbim7kdffvtosuldtv5m3uy4 Status.AMBIGUOUS OK.DUMMY
+
+Different.
+
 * [ ] https://fatcat.wiki/release/dauh7n5w65enhk5zwdfwqxv344 https://fatcat.wiki/release/773m6wdunreqzlae6nts44rudy Status.AMBIGUOUS OK.DUMMY
 * [ ] https://fatcat.wiki/release/voruupqxhvggfex4zlczcmjxxu https://fatcat.wiki/release/jg72qhdvmncfdfxg5l47hw3uba Status.AMBIGUOUS OK.DUMMY
 * [ ] https://fatcat.wiki/release/nc5qyc3umff5zevew2dobmispy https://fatcat.wiki/release/frdluoflhfgglphbotrazdyioq Status.AMBIGUOUS OK.DUMMY
diff --git a/tests/data/release/3f3yt32rrbdzdewtcyrcyihgju b/tests/data/release/3f3yt32rrbdzdewtcyrcyihgju
new file mode 100644
index 0000000..2feb19b
--- /dev/null
+++ b/tests/data/release/3f3yt32rrbdzdewtcyrcyihgju
@@ -0,0 +1,60 @@
+{
+  "abstracts": [],
+  "container_id": "ud7h2fmzczbshoux3s2pzgtf2q",
+  "contribs": [
+    {
+      "extra": {
+        "seq": "first"
+      },
+      "index": 0,
+      "raw_name": "Kuidong Xu",
+      "role": "author"
+    },
+    {
+      "index": 1,
+      "raw_name": "Joong Ki Choi",
+      "role": "author"
+    },
+    {
+      "index": 2,
+      "raw_name": "Eun Jin Yang",
+      "role": "author"
+    },
+    {
+      "index": 3,
+      "raw_name": "Kyu Chul Lee",
+      "role": "author"
+    },
+    {
+      "index": 4,
+      "raw_name": "Yanli Lei",
+      "role": "author"
+    }
+  ],
+  "ext_ids": {
+    "doi": "10.1016/s0025-326x(02)00090-5",
+    "pmid": "12405212",
+    "wikidata_qid": "Q53142806"
+  },
+  "extra": {
+    "crossref": {
+      "alternative-id": [
+        "S0025326X02000905"
+      ],
+      "type": "journal-article"
+    }
+  },
+  "ident": "3f3yt32rrbdzdewtcyrcyihgju",
+  "language": "en",
+  "pages": "877-886",
+  "publisher": "Elsevier BV",
+  "refs": [],
+  "release_stage": "published",
+  "release_type": "article-journal",
+  "release_year": 2002,
+  "revision": "2d34fb3a-2b9b-4f4b-9748-ac77a7c774da",
+  "state": "active",
+  "title": "Biomonitoring of coastal pollution status using protozoan communities with a modified PFU method",
+  "volume": "44",
+  "work_id": "usqag3t54jaghddclwakg7tgaq"
+}
diff --git a/tests/data/release/b7bbygyawzdsthai7j7rmztrxe b/tests/data/release/b7bbygyawzdsthai7j7rmztrxe
new file mode 100644
index 0000000..701f25c
--- /dev/null
+++ b/tests/data/release/b7bbygyawzdsthai7j7rmztrxe
@@ -0,0 +1,32 @@
+{
+  "abstracts": [],
+  "container_id": "mbtcy2nedvhizi6blcfehl2zaa",
+  "contribs": [],
+  "ext_ids": {
+    "jstor": "44062975"
+  },
+  "extra": {
+    "jstor": {
+      "issue_id": "i40168501",
+      "journal_ids": [
+        "jeducation",
+        "j50009610"
+      ]
+    }
+  },
+  "ident": "b7bbygyawzdsthai7j7rmztrxe",
+  "issue": "21 (1355)",
+  "language": "en",
+  "pages": "353",
+  "publisher": "New England Publishing Company",
+  "refs": [],
+  "release_date": "1901-11-28",
+  "release_stage": "published",
+  "release_type": "stub",
+  "release_year": 1901,
+  "revision": "e0b8f83c-d4b5-4ab2-87f8-a86da76fcb66",
+  "state": "active",
+  "title": "A NEW AID IN SCHOOL WORK",
+  "volume": "54",
+  "work_id": "qjgdl3apf5d45eh5f3wdab3gma"
+}
diff --git a/tests/data/release/hxmxijrd3fgodd3q3crgt7d3c4 b/tests/data/release/hxmxijrd3fgodd3q3crgt7d3c4
new file mode 100644
index 0000000..9d0593b
--- /dev/null
+++ b/tests/data/release/hxmxijrd3fgodd3q3crgt7d3c4
@@ -0,0 +1,51 @@
+{
+  "abstracts": [],
+  "contribs": [
+    {
+      "extra": {
+        "seq": "first"
+      },
+      "index": 0,
+      "raw_name": "J.K. Choi",
+      "role": "author"
+    },
+    {
+      "index": 1,
+      "raw_name": "K. Xu",
+      "role": "author"
+    },
+    {
+      "index": 2,
+      "raw_name": "E.J. Yang",
+      "role": "author"
+    },
+    {
+      "index": 3,
+      "raw_name": "K.C. Lee",
+      "role": "author"
+    },
+    {
+      "index": 4,
+      "raw_name": "Y. Lei",
+      "role": "author"
+    }
+  ],
+  "ext_ids": {
+    "doi": "10.1109/oceans.2003.178350"
+  },
+  "extra": {
+    "container_name": "Oceans 2003. Celebrating the Past ... Teaming Toward the Future (IEEE Cat. No.03CH37492)",
+    "crossref": {
+      "type": "proceedings-article"
+    }
+  },
+  "ident": "hxmxijrd3fgodd3q3crgt7d3c4",
+  "publisher": "IEEE",
+  "refs": [],
+  "release_type": "paper-conference",
+  "release_year": 2003,
+  "revision": "e049728c-a800-472f-9001-b5a09d556761",
+  "state": "active",
+  "title": "Biomonitoring of coastal pollution status using protozoan communities with a modified PFU method",
+  "work_id": "mlhtnjhxefeudk3ya57ksvxqim"
+}
diff --git a/tests/data/release/mvvbim7kdffvtosuldtv5m3uy4 b/tests/data/release/mvvbim7kdffvtosuldtv5m3uy4
new file mode 100644
index 0000000..9c06066
--- /dev/null
+++ b/tests/data/release/mvvbim7kdffvtosuldtv5m3uy4
@@ -0,0 +1,32 @@
+{
+  "abstracts": [],
+  "container_id": "mbtcy2nedvhizi6blcfehl2zaa",
+  "contribs": [],
+  "ext_ids": {
+    "jstor": "44054275"
+  },
+  "extra": {
+    "jstor": {
+      "issue_id": "i40168274",
+      "journal_ids": [
+        "jeducation",
+        "j50009610"
+      ]
+    }
+  },
+  "ident": "mvvbim7kdffvtosuldtv5m3uy4",
+  "issue": "22 (1356)",
+  "language": "en",
+  "pages": "383",
+  "publisher": "New England Publishing Company",
+  "refs": [],
+  "release_date": "1901-12-05",
+  "release_stage": "published",
+  "release_type": "stub",
+  "release_year": 1901,
+  "revision": "afeb3f72-484f-4459-9d52-c90ba316eefb",
+  "state": "active",
+  "title": "A NEW AID IN SCHOOL WORK",
+  "volume": "54",
+  "work_id": "v4yyq7zb7zc73jt7mpkhrczici"
+}
diff --git a/tests/data/verify.csv b/tests/data/verify.csv
index 1ea544b..8de9395 100644
--- a/tests/data/verify.csv
+++ b/tests/data/verify.csv
@@ -130,3 +130,5 @@ tmlg73royrdwdhl6nijf6m7vzy,3w4tibll4rdernjrn4hkkyqsem,Status.DIFFERENT,Miss.CONT
 eqcgtpav3na5jh56o5vjsvb4ei,7x7tszf54zggvp4xkrhakp667u,Status.STRONG,OK.PMID_DOI_PAIR
 wm2p5fznwffknjx56lvmr7hn4q,idpgijvcsnbqrgs2dg36vzzdzm,Status.DIFFERENT,Miss.SHARED_DOI_PREFIX
 72uzveph65ce7kfdct2wpgh5j4,nqcfu4il45aixekvk3rwflahdm,Status.DIFFERENT,
+mvvbim7kdffvtosuldtv5m3uy4,b7bbygyawzdsthai7j7rmztrxe,Status.DIFFERENT,
+3f3yt32rrbdzdewtcyrcyihgju,hxmxijrd3fgodd3q3crgt7d3c4,Status.STRONG,
diff --git a/tests/test_verify.py b/tests/test_verify.py
index 75b1190..5a60326 100644
--- a/tests/test_verify.py
+++ b/tests/test_verify.py
@@ -38,7 +38,7 @@ def test_compare():
             try:
                 a, b, expected_status, expected_reason = row
             except ValueError as exc:
-                pytest.fail("invalid test file, maybe missing a comma? {}".format(exc))
+                pytest.fail("invalid test file, maybe too many (or few) commas in row {}? {}".format(i + 1, exc))
             status, reason = compare(load_release_ident(a), load_release_ident(b))
             if not expected_status or expected_status.lower() == "todo":
                 logger.warning(