aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fuzzycat/grobid_unstructured.py3
-rw-r--r--fuzzycat/simple.py3
-rw-r--r--fuzzycat/utils.py14
-rw-r--r--fuzzycat/verify.py6
-rw-r--r--tests/test_utils.py15
5 files changed, 35 insertions, 6 deletions
diff --git a/fuzzycat/grobid_unstructured.py b/fuzzycat/grobid_unstructured.py
index 79c39d3..5462ae1 100644
--- a/fuzzycat/grobid_unstructured.py
+++ b/fuzzycat/grobid_unstructured.py
@@ -18,6 +18,7 @@ from fatcat_openapi_client import ReleaseContrib, ReleaseEntity, ReleaseExtIds
from fuzzycat.config import settings
from fuzzycat.grobid2json import biblio_info
+from fuzzycat.utils import clean_doi
GROBID_API_BASE = settings.get("GROBID_API_BASE", "https://grobid.qa.fatcat.wiki")
@@ -89,7 +90,7 @@ def grobid_ref_to_release(ref: dict) -> ReleaseEntity:
issue=ref.get("issue"),
pages=ref.get("pages"),
ext_ids=ReleaseExtIds(
- doi=ref.get("doi"),
+ doi=clean_doi(ref.get("doi")),
pmid=ref.get("pmid"),
pmcid=ref.get("pmcid"),
arxiv=ref.get("arxiv_id"),
diff --git a/fuzzycat/simple.py b/fuzzycat/simple.py
index c78ac28..8b206b1 100644
--- a/fuzzycat/simple.py
+++ b/fuzzycat/simple.py
@@ -26,6 +26,7 @@ from fuzzycat.entities import entity_to_dict
from fuzzycat.grobid_unstructured import grobid_parse_unstructured
from fuzzycat.matching import match_release_fuzzy
from fuzzycat.verify import verify
+from fuzzycat.utils import clean_doi
@dataclass
@@ -184,7 +185,7 @@ def biblio_to_release(biblio: dict) -> ReleaseEntity:
release = ReleaseEntity(
title=biblio.get("title"),
ext_ids=ReleaseExtIds(
- doi=biblio.get("doi"),
+ doi=clean_doi(biblio.get("doi")),
pmid=biblio.get("pmid"),
pmcid=biblio.get("pmcid"),
arxiv=biblio.get("arxiv_id"),
diff --git a/fuzzycat/utils.py b/fuzzycat/utils.py
index bdca7b6..a1c5124 100644
--- a/fuzzycat/utils.py
+++ b/fuzzycat/utils.py
@@ -6,6 +6,7 @@ import re
import string
import subprocess
import tempfile
+from typing import Optional
import requests
from glom import PathAccessError, glom
@@ -80,6 +81,19 @@ def dict_key_exists(doc, path):
else:
return True
+def clean_doi(raw: Optional[str]) -> Optional[str]:
+ if not raw:
+ return None
+ raw = raw.strip().lower()
+ if raw.startswith("doi:"):
+ raw = raw[4:]
+ if not "10." in raw:
+ return None
+ if not raw.startswith("10."):
+ raw = raw[raw.find("10."):]
+ if raw[7:9] == "//":
+ raw = raw[:8] + raw[9:]
+ return raw
def doi_prefix(v):
"""
diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py
index f32121d..1eeea40 100644
--- a/fuzzycat/verify.py
+++ b/fuzzycat/verify.py
@@ -92,7 +92,7 @@ from fuzzycat.data import (CONTAINER_NAME_BLACKLIST, PUBLISHER_BLACKLIST, TITLE_
from fuzzycat.entities import entity_to_dict
from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, dict_key_exists,
doi_prefix, has_doi_prefix, jaccard, num_project, parse_page_string,
- slugify_string)
+ slugify_string, clean_doi)
Verify = collections.namedtuple("Verify", "status reason")
@@ -167,8 +167,8 @@ def verify(a: Dict, b: Dict, min_title_length=5) -> Tuple[str, str]:
# A few items have the same DOI.
try:
- a_doi = glom(a, "ext_ids.doi")
- b_doi = glom(b, "ext_ids.doi")
+ a_doi = clean_doi(glom(a, "ext_ids.doi"))
+ b_doi = clean_doi(glom(b, "ext_ids.doi"))
if a_doi is not None and a_doi == b_doi:
return Verify(Status.EXACT, Reason.DOI)
except PathAccessError:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index d0bcfc1..21b85a4 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,7 +3,7 @@ import os
from fuzzycat.utils import (author_similarity_score, cut, jaccard, nwise, slugify_string,
token_n_grams, tokenize_string, parse_page_string, dict_key_exists,
- zstdlines, es_compat_hits_total)
+ zstdlines, es_compat_hits_total, clean_doi)
def test_slugify_string():
@@ -123,3 +123,16 @@ def test_es_compat_hits_total():
)
for r, expected in cases:
assert es_compat_hits_total(r) == expected
+
+def test_clean_doi():
+ assert clean_doi(None) == None
+ assert clean_doi("blah") == None
+ assert clean_doi("10.1234/asdf ") == "10.1234/asdf"
+ assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+ assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+ assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf"
+ # GROBID mangled DOI
+ assert clean_doi("21924DOI10.1234/asdf ") == "10.1234/asdf"
+ assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf"
+ assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf"
+ assert clean_doi("10.7326/M20-6817") == "10.7326/m20-6817"