diff options
| -rw-r--r-- | fatcat_scholar/identifiers.py | 90 | ||||
| -rw-r--r-- | fatcat_scholar/search.py | 25 |
2 files changed, 106 insertions, 9 deletions
diff --git a/fatcat_scholar/identifiers.py b/fatcat_scholar/identifiers.py new file mode 100644 index 0000000..2ea09c7 --- /dev/null +++ b/fatcat_scholar/identifiers.py @@ -0,0 +1,90 @@ +import re +from typing import Optional + +DOI_REGEX = re.compile(r"^10.\d{3,6}/\S+$") + + +def clean_doi(raw: Optional[str]) -> Optional[str]: + """ + Removes any: + - padding whitespace + - 'doi:' prefix + - URL prefix + + Does not try to un-URL-encode + + Returns None if not a valid DOI + """ + if not raw: + return None + raw = raw.strip() + if "\u2013" in raw: + # Do not attempt to normalize "en dash" and since FC does not allow + # unicode in DOI, treat this as invalid. + return None + if len(raw.split()) != 1: + return None + if raw.startswith("doi:"): + raw = raw[4:] + if raw.startswith("http://"): + raw = raw[7:] + if raw.startswith("https://"): + raw = raw[8:] + if raw.startswith("doi.org/"): + raw = raw[8:] + if raw.startswith("dx.doi.org/"): + raw = raw[11:] + if raw[7:9] == "//": + raw = raw[:8] + raw[9:] + + # fatcatd uses same REGEX, but Rust regex rejects these characters, while + # python doesn't. DOIs are syntaxtually valid, but very likely to be typos; + # for now filter them out. + for c in ("¬",): + if c in raw: + return None + + if not raw.startswith("10."): + return None + if not DOI_REGEX.fullmatch(raw): + return None + # will likely want to expand DOI_REGEX to exclude non-ASCII characters, but + # for now block specific characters so we can get PubMed importer running + # again. + if "ä" in raw: + return None + return raw + + +def test_clean_doi() -> None: + assert clean_doi("10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("10.1037//0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.1037/0002-9432.72.1.50") == "10.1037/0002-9432.72.1.50" + assert clean_doi("10.23750/abm.v88i2 -s.6506") == None + assert clean_doi("10.17167/mksz.2017.2.129–155") == None + assert clean_doi("http://doi.org/10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("https://dx.doi.org/10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("doi:10.1234/asdf ") == "10.1234/asdf" + assert clean_doi("doi:10.1234/ asdf ") == None + assert clean_doi("10.4149/gpb¬_2017042") == None # "logical negation" character + assert ( + clean_doi("10.6002/ect.2020.häyry") == None + ) # this example via pubmed (pmid:32519616) + + +def clean_pmcid(raw: Optional[str]) -> Optional[str]: + if not raw: + return None + raw = raw.strip() + if len(raw.split()) != 1: + return None + if raw.startswith("PMC") and raw[3:] and raw[3:].isdigit(): + return raw + return None + + +def test_clean_pmcid() -> None: + assert clean_pmcid("10.1234/asdf ") == None + assert clean_pmcid("") == None + assert clean_pmcid(None) == None + assert clean_pmcid("PMC123") == "PMC123" diff --git a/fatcat_scholar/search.py b/fatcat_scholar/search.py index 0b6798d..d29a720 100644 --- a/fatcat_scholar/search.py +++ b/fatcat_scholar/search.py @@ -16,6 +16,8 @@ from pydantic import BaseModel # pytype: enable=import-error +from fatcat_scholar.identifiers import * + # i18n note: the use of gettext below doesn't actually do the translation here, # it just ensures that the strings are caught by babel for translation later @@ -97,15 +99,20 @@ def do_fulltext_search( search = Search(using=es_client, index=settings.ELASTICSEARCH_FULLTEXT_INDEX) - # Convert raw DOIs to DOI queries - if ( - query.q - and len(query.q.split()) == 1 - and query.q.startswith("10.") - and query.q.count("/") >= 1 - ): - search = search.filter("terms", doi=query.q) - query.q = "*" + # Try handling raw identifier queries + if query.q and len(query.q.strip().split()) == 1 and not '"' in query.q: + doi = clean_doi(query.q) + if doi: + query.q = f'doi:"{doi}"' + query.filter_type = "everything" + query.filter_availability = "everything" + query.filter_time = "all_time" + pmcid = clean_pmcid(query.q) + if pmcid: + query.q = f'pmcid:"{pmcid}"' + query.filter_type = "everything" + query.filter_availability = "everything" + query.filter_time = "all_time" # type filters if query.filter_type == "papers" or query.filter_type is None: |