From dcb43953784db9a8068e5c51d4499592a49f2ea8 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 23 Dec 2019 18:30:52 -0800 Subject: doi parsing fixes Replace emdash with regular dash. Replace double slash after partner ID with single slash. This conversion seems to be done by crossref automatically on lookup. I tried several examples, using doi.org resolver and Crossref API lookup. Note that there are a number of fatcat entities with '//' in the DOI. --- python/fatcat_tools/normal.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'python') diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py index 1cc66633..a77c5eb0 100644 --- a/python/fatcat_tools/normal.py +++ b/python/fatcat_tools/normal.py @@ -22,6 +22,7 @@ def clean_doi(raw): if not raw: return None raw = raw.strip() + raw = raw.replace('\u2013', '-') # emdash if len(raw.split()) != 1: return None if raw.startswith("doi:"): @@ -34,6 +35,8 @@ def clean_doi(raw): raw = raw[8:] if raw.startswith("dx.doi.org/"): raw = raw[11:] + if raw[7:9] == "//": + raw = raw[:8] + raw[9:] if not raw.startswith("10."): return None if not DOI_REGEX.fullmatch(raw): @@ -42,6 +45,10 @@ def clean_doi(raw): def test_clean_doi(): assert clean_doi("10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.23750/abm.v88i2 -s.6506") == None + assert clean_doi("10.17167/mksz.2017.2.129–155") == "10.17167/mksz.2017.2.129-155" assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf" assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf" -- cgit v1.2.3