DOI clean/normalize helper; and use in verification etc

1 files changed, 14 insertions, 1 deletions

diff --git a/tests/test_utils.py b/tests/test_utils.py
index d0bcfc1..21b85a4 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,7 +3,7 @@ import os
 
 from fuzzycat.utils import (author_similarity_score, cut, jaccard, nwise, slugify_string,
                             token_n_grams, tokenize_string, parse_page_string, dict_key_exists,
-                            zstdlines, es_compat_hits_total)
+                            zstdlines, es_compat_hits_total, clean_doi)
 
 
 def test_slugify_string():
@@ -123,3 +123,16 @@ def test_es_compat_hits_total():
     )
     for r, expected in cases:
         assert es_compat_hits_total(r) == expected
+
+def test_clean_doi():
+    assert clean_doi(None) == None
+    assert clean_doi("blah") == None
+    assert clean_doi("10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+    assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+    assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf"
+    # GROBID mangled DOI
+    assert clean_doi("21924DOI10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf"
+    assert clean_doi("10.7326/M20-6817") == "10.7326/m20-6817"