DOI clean/normalize helper; and use in verification etc

5 files changed, 35 insertions, 6 deletions

diff --git a/fuzzycat/grobid_unstructured.py b/fuzzycat/grobid_unstructured.py
index 79c39d3..5462ae1 100644
--- a/fuzzycat/grobid_unstructured.py
+++ b/fuzzycat/grobid_unstructured.py
@@ -18,6 +18,7 @@ from fatcat_openapi_client import ReleaseContrib, ReleaseEntity, ReleaseExtIds
 
 from fuzzycat.config import settings
 from fuzzycat.grobid2json import biblio_info
+from fuzzycat.utils import clean_doi
 
 GROBID_API_BASE = settings.get("GROBID_API_BASE", "https://grobid.qa.fatcat.wiki")
 
@@ -89,7 +90,7 @@ def grobid_ref_to_release(ref: dict) -> ReleaseEntity:
         issue=ref.get("issue"),
         pages=ref.get("pages"),
         ext_ids=ReleaseExtIds(
-            doi=ref.get("doi"),
+            doi=clean_doi(ref.get("doi")),
             pmid=ref.get("pmid"),
             pmcid=ref.get("pmcid"),
             arxiv=ref.get("arxiv_id"),
diff --git a/fuzzycat/simple.py b/fuzzycat/simple.py
index c78ac28..8b206b1 100644
--- a/fuzzycat/simple.py
+++ b/fuzzycat/simple.py
@@ -26,6 +26,7 @@ from fuzzycat.entities import entity_to_dict
 from fuzzycat.grobid_unstructured import grobid_parse_unstructured
 from fuzzycat.matching import match_release_fuzzy
 from fuzzycat.verify import verify
+from fuzzycat.utils import clean_doi
 
 
 @dataclass
@@ -184,7 +185,7 @@ def biblio_to_release(biblio: dict) -> ReleaseEntity:
     release = ReleaseEntity(
         title=biblio.get("title"),
         ext_ids=ReleaseExtIds(
-            doi=biblio.get("doi"),
+            doi=clean_doi(biblio.get("doi")),
             pmid=biblio.get("pmid"),
             pmcid=biblio.get("pmcid"),
             arxiv=biblio.get("arxiv_id"),
diff --git a/fuzzycat/utils.py b/fuzzycat/utils.py
index bdca7b6..a1c5124 100644
--- a/fuzzycat/utils.py
+++ b/fuzzycat/utils.py
@@ -6,6 +6,7 @@ import re
 import string
 import subprocess
 import tempfile
+from typing import Optional
 
 import requests
 from glom import PathAccessError, glom
@@ -80,6 +81,19 @@ def dict_key_exists(doc, path):
     else:
         return True
 
+def clean_doi(raw: Optional[str]) -> Optional[str]:
+    if not raw:
+        return None
+    raw = raw.strip().lower()
+    if raw.startswith("doi:"):
+        raw = raw[4:]
+    if not "10." in raw:
+        return None
+    if not raw.startswith("10."):
+        raw = raw[raw.find("10."):]
+    if raw[7:9] == "//":
+        raw = raw[:8] + raw[9:]
+    return raw
 
 def doi_prefix(v):
     """
diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py
index f32121d..1eeea40 100644
--- a/fuzzycat/verify.py
+++ b/fuzzycat/verify.py
@@ -92,7 +92,7 @@ from fuzzycat.data import (CONTAINER_NAME_BLACKLIST, PUBLISHER_BLACKLIST, TITLE_
 from fuzzycat.entities import entity_to_dict
 from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, dict_key_exists,
                             doi_prefix, has_doi_prefix, jaccard, num_project, parse_page_string,
-                            slugify_string)
+                            slugify_string, clean_doi)
 
 Verify = collections.namedtuple("Verify", "status reason")
 
@@ -167,8 +167,8 @@ def verify(a: Dict, b: Dict, min_title_length=5) -> Tuple[str, str]:
 
     # A few items have the same DOI.
     try:
-        a_doi = glom(a, "ext_ids.doi")
-        b_doi = glom(b, "ext_ids.doi")
+        a_doi = clean_doi(glom(a, "ext_ids.doi"))
+        b_doi = clean_doi(glom(b, "ext_ids.doi"))
         if a_doi is not None and a_doi == b_doi:
             return Verify(Status.EXACT, Reason.DOI)
     except PathAccessError:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index d0bcfc1..21b85a4 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,7 +3,7 @@ import os
 
 from fuzzycat.utils import (author_similarity_score, cut, jaccard, nwise, slugify_string,
                             token_n_grams, tokenize_string, parse_page_string, dict_key_exists,
-                            zstdlines, es_compat_hits_total)
+                            zstdlines, es_compat_hits_total, clean_doi)
 
 
 def test_slugify_string():
@@ -123,3 +123,16 @@ def test_es_compat_hits_total():
     )
     for r, expected in cases:
         assert es_compat_hits_total(r) == expected
+
+def test_clean_doi():
+    assert clean_doi(None) == None
+    assert clean_doi("blah") == None
+    assert clean_doi("10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+    assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+    assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf"
+    # GROBID mangled DOI
+    assert clean_doi("21924DOI10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("10.7326/M20-6817") == "10.7326/m20-6817"