DOI clean/normalize helper; and use in verification etc

4 files changed, 21 insertions, 5 deletions

diff --git a/fuzzycat/grobid_unstructured.py b/fuzzycat/grobid_unstructured.py
index 79c39d3..5462ae1 100644
--- a/fuzzycat/grobid_unstructured.py
+++ b/fuzzycat/grobid_unstructured.py
@@ -18,6 +18,7 @@ from fatcat_openapi_client import ReleaseContrib, ReleaseEntity, ReleaseExtIds
 
 from fuzzycat.config import settings
 from fuzzycat.grobid2json import biblio_info
+from fuzzycat.utils import clean_doi
 
 GROBID_API_BASE = settings.get("GROBID_API_BASE", "https://grobid.qa.fatcat.wiki")
 
@@ -89,7 +90,7 @@ def grobid_ref_to_release(ref: dict) -> ReleaseEntity:
         issue=ref.get("issue"),
         pages=ref.get("pages"),
         ext_ids=ReleaseExtIds(
-            doi=ref.get("doi"),
+            doi=clean_doi(ref.get("doi")),
             pmid=ref.get("pmid"),
             pmcid=ref.get("pmcid"),
             arxiv=ref.get("arxiv_id"),
diff --git a/fuzzycat/simple.py b/fuzzycat/simple.py
index c78ac28..8b206b1 100644
--- a/fuzzycat/simple.py
+++ b/fuzzycat/simple.py
@@ -26,6 +26,7 @@ from fuzzycat.entities import entity_to_dict
 from fuzzycat.grobid_unstructured import grobid_parse_unstructured
 from fuzzycat.matching import match_release_fuzzy
 from fuzzycat.verify import verify
+from fuzzycat.utils import clean_doi
 
 
 @dataclass
@@ -184,7 +185,7 @@ def biblio_to_release(biblio: dict) -> ReleaseEntity:
     release = ReleaseEntity(
         title=biblio.get("title"),
         ext_ids=ReleaseExtIds(
-            doi=biblio.get("doi"),
+            doi=clean_doi(biblio.get("doi")),
             pmid=biblio.get("pmid"),
             pmcid=biblio.get("pmcid"),
             arxiv=biblio.get("arxiv_id"),
diff --git a/fuzzycat/utils.py b/fuzzycat/utils.py
index bdca7b6..a1c5124 100644
--- a/fuzzycat/utils.py
+++ b/fuzzycat/utils.py
@@ -6,6 +6,7 @@ import re
 import string
 import subprocess
 import tempfile
+from typing import Optional
 
 import requests
 from glom import PathAccessError, glom
@@ -80,6 +81,19 @@ def dict_key_exists(doc, path):
     else:
         return True
 
+def clean_doi(raw: Optional[str]) -> Optional[str]:
+    if not raw:
+        return None
+    raw = raw.strip().lower()
+    if raw.startswith("doi:"):
+        raw = raw[4:]
+    if not "10." in raw:
+        return None
+    if not raw.startswith("10."):
+        raw = raw[raw.find("10."):]
+    if raw[7:9] == "//":
+        raw = raw[:8] + raw[9:]
+    return raw
 
 def doi_prefix(v):
     """
diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py
index f32121d..1eeea40 100644
--- a/fuzzycat/verify.py
+++ b/fuzzycat/verify.py
@@ -92,7 +92,7 @@ from fuzzycat.data import (CONTAINER_NAME_BLACKLIST, PUBLISHER_BLACKLIST, TITLE_
 from fuzzycat.entities import entity_to_dict
 from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, dict_key_exists,
                             doi_prefix, has_doi_prefix, jaccard, num_project, parse_page_string,
-                            slugify_string)
+                            slugify_string, clean_doi)
 
 Verify = collections.namedtuple("Verify", "status reason")
 
@@ -167,8 +167,8 @@ def verify(a: Dict, b: Dict, min_title_length=5) -> Tuple[str, str]:
 
     # A few items have the same DOI.
     try:
-        a_doi = glom(a, "ext_ids.doi")
-        b_doi = glom(b, "ext_ids.doi")
+        a_doi = clean_doi(glom(a, "ext_ids.doi"))
+        b_doi = clean_doi(glom(b, "ext_ids.doi"))
         if a_doi is not None and a_doi == b_doi:
             return Verify(Status.EXACT, Reason.DOI)
     except PathAccessError: