From 53756811572bab0679cb8cee1b9de95e7b29b96a Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Fri, 17 Jan 2020 14:03:00 +0100 Subject: do not normalize "en dash" in DOI Technically, [...] DOI names may incorporate any printable characters from the Universal Character Set (UCS-2), of ISO/IEC 10646, which is the character set defined by Unicode (https://www.doi.org/doi_handbook/2_Numbering.html#2.5.1). For mostly QA reasons, we currently treat a DOI with an "en dash" as invalid. --- python/fatcat_tools/normal.py | 7 +++++-- python/tests/files/datacite/datacite_doc_23.json | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py index a77c5eb0..7b4bd19c 100644 --- a/python/fatcat_tools/normal.py +++ b/python/fatcat_tools/normal.py @@ -22,7 +22,10 @@ def clean_doi(raw): if not raw: return None raw = raw.strip() - raw = raw.replace('\u2013', '-') # emdash + if '\u2013' in raw: + # Do not attempt to normalize "en dash" and since FC does not allow + # unicode in DOI, treat this as invalid. + return None if len(raw.split()) != 1: return None if raw.startswith("doi:"): @@ -48,7 +51,7 @@ def test_clean_doi(): assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" assert clean_doi("10.23750/abm.v88i2 -s.6506") == None - assert clean_doi("10.17167/mksz.2017.2.129–155") == "10.17167/mksz.2017.2.129-155" + assert clean_doi("10.17167/mksz.2017.2.129–155") == None assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf" diff --git a/python/tests/files/datacite/datacite_doc_23.json b/python/tests/files/datacite/datacite_doc_23.json index b755f1a5..8eaa8c21 100644 --- a/python/tests/files/datacite/datacite_doc_23.json +++ b/python/tests/files/datacite/datacite_doc_23.json @@ -23,7 +23,7 @@ "dateType": "Issued" } ], - "doi": "10.7916/d86x0cg1–xxx", + "doi": "10.7916/d86x0cg1-xxx", "isActive": true, "language": "GERMAN", "publicationYear": 2017, -- cgit v1.2.3