aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-12-23 18:30:52 -0800
committerBryan Newbold <bnewbold@robocracy.org>2019-12-23 18:30:54 -0800
commitdcb43953784db9a8068e5c51d4499592a49f2ea8 (patch)
tree682325701be219f1ee282e8198c7ec927cced8f9
parentaa6bec5993e5444937054022b46d751645d0183c (diff)
downloadfatcat-dcb43953784db9a8068e5c51d4499592a49f2ea8.tar.gz
fatcat-dcb43953784db9a8068e5c51d4499592a49f2ea8.zip
doi parsing fixes
Replace emdash with regular dash. Replace double slash after partner ID with single slash. This conversion seems to be done by crossref automatically on lookup. I tried several examples, using doi.org resolver and Crossref API lookup. Note that there are a number of fatcat entities with '//' in the DOI.
-rw-r--r--python/fatcat_tools/normal.py7
1 files changed, 7 insertions, 0 deletions
diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py
index 1cc66633..a77c5eb0 100644
--- a/python/fatcat_tools/normal.py
+++ b/python/fatcat_tools/normal.py
@@ -22,6 +22,7 @@ def clean_doi(raw):
if not raw:
return None
raw = raw.strip()
+ raw = raw.replace('\u2013', '-') # emdash
if len(raw.split()) != 1:
return None
if raw.startswith("doi:"):
@@ -34,6 +35,8 @@ def clean_doi(raw):
raw = raw[8:]
if raw.startswith("dx.doi.org/"):
raw = raw[11:]
+ if raw[7:9] == "//":
+ raw = raw[:8] + raw[9:]
if not raw.startswith("10."):
return None
if not DOI_REGEX.fullmatch(raw):
@@ -42,6 +45,10 @@ def clean_doi(raw):
def test_clean_doi():
assert clean_doi("10.1234/asdf ") == "10.1234/asdf"
+ assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+ assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50"
+ assert clean_doi("10.23750/abm.v88i2 -s.6506") == None
+ assert clean_doi("10.17167/mksz.2017.2.129–155") == "10.17167/mksz.2017.2.129-155"
assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf"
assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf"
assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf"