diff options
Diffstat (limited to 'fatcat_scholar')
| -rw-r--r-- | fatcat_scholar/identifiers.py | 2 | ||||
| -rw-r--r-- | fatcat_scholar/schema.py | 13 | ||||
| -rw-r--r-- | fatcat_scholar/search.py | 3 | ||||
| -rw-r--r-- | fatcat_scholar/templates/access_404.html | 35 | ||||
| -rw-r--r-- | fatcat_scholar/templates/search_macros.html | 3 | ||||
| -rw-r--r-- | fatcat_scholar/transform.py | 142 | ||||
| -rw-r--r-- | fatcat_scholar/web.py | 226 |
7 files changed, 317 insertions, 107 deletions
diff --git a/fatcat_scholar/identifiers.py b/fatcat_scholar/identifiers.py index 7572e20..9a64de8 100644 --- a/fatcat_scholar/identifiers.py +++ b/fatcat_scholar/identifiers.py @@ -27,7 +27,7 @@ def clean_doi(raw: Optional[str]) -> Optional[str]: if not "10." in raw: return None if not raw.startswith("10."): - raw = raw[raw.find("10."):] + raw = raw[raw.find("10.") :] if raw[7:9] == "//": raw = raw[:8] + raw[9:] diff --git a/fatcat_scholar/schema.py b/fatcat_scholar/schema.py index e6d0422..0fcf56e 100644 --- a/fatcat_scholar/schema.py +++ b/fatcat_scholar/schema.py @@ -270,11 +270,12 @@ class RefBiblio(BaseModel): volume: Optional[str] issue: Optional[str] pages: Optional[str] + version: Optional[str] doi: Optional[str] pmid: Optional[str] pmcid: Optional[str] arxiv_id: Optional[str] - isbn13: Optional[str] + isbn: Optional[str] url: Optional[str] @@ -284,7 +285,7 @@ class RefStructured(BaseModel): work_ident: Optional[str] release_stage: Optional[str] release_year: Optional[int] - index: Optional[int] + index: Optional[int] # 1-indexed key: Optional[str] locator: Optional[str] target_release_id: Optional[str] @@ -300,9 +301,12 @@ class RefTarget(BaseModel): def clean_small_int(raw: Optional[str]) -> Optional[int]: - if not raw or not raw.isdigit(): + if not raw or not raw.strip().isdigit(): + return None + try: + val = int(raw.strip()) + except ValueError: return None - val = int(raw) if abs(val) > 30000: return None return val @@ -317,6 +321,7 @@ def test_clean_small_int() -> None: assert clean_small_int("1200003") == None assert clean_small_int("-123") == None assert clean_small_int("48844") == None + assert clean_small_int("1990²") == None def doi_split_prefix(doi: str) -> str: diff --git a/fatcat_scholar/search.py b/fatcat_scholar/search.py index 121cb69..dccaf07 100644 --- a/fatcat_scholar/search.py +++ b/fatcat_scholar/search.py @@ -377,6 +377,9 @@ def do_fulltext_search( search = search.params(track_total_hits=True) search = search[offset : (offset + limit)] + if settings.ELASTICSEARCH_QUERY_PREFERENCE: + search = search.params(preference=settings.ELASTICSEARCH_QUERY_PREFERENCE) + query_start = datetime.datetime.now() try: resp = search.execute() diff --git a/fatcat_scholar/templates/access_404.html b/fatcat_scholar/templates/access_404.html new file mode 100644 index 0000000..d058186 --- /dev/null +++ b/fatcat_scholar/templates/access_404.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block title %} +404 - {{ super() }} +{% endblock %} + +{% block main %} +<div class="ui icon error message"> + <div class="content"> + <div class="header">{% trans %}404: Access Location Not Found{% endtrans %}</div> + <p>{% trans %}We could not find a valid redirect for the URL you tried. Sorry about that!{% endtrans %} + <p>{% trans %}There may be a typo, truncation, or encoding error. Or, the resource may have been removed from our catalog.{% endtrans %} + <p>{% trans %}Some places you can visit try to hunt down this resource (or a replacement) include:{% endtrans %} + <ul> + {% if original_url %} + <li>{% trans %}Original web url:{% endtrans %} + <br> + <code style="word-break: break-all;"><a href="{{ original_url }}">{{ original_url }}</a></code> + </li> + <li><a href="https://web.archive.org/web/*/{{ original_url }}">{% trans %}Wayback Machine calendar page (all captures){% endtrans %}</a> + {% endif %} + {% if archiveorg_path %} + <li>{% trans %}archive.org download link for the item:{% endtrans %} + {% set archiveorg_url="https://archive.org/download" + archiveorg_path %} + <br> + <code style="word-break: break-all;"><a href="{{ archiveorg_url }}">{{ archiveorg_url }}</a></code> + {% endif %} + {% if work_ident %} + <li><a href="/work/{{ work_ident }}">{% trans %}Scholar landing page{% endtrans %}</a> + <li><a href="https://fatcat.wiki/work/{{ work_ident }}">{% trans %}Fatcat catalog page{% endtrans %}</a> + {% endif %} + </ul> + </div> +</div> +{% endblock %} diff --git a/fatcat_scholar/templates/search_macros.html b/fatcat_scholar/templates/search_macros.html index 4965045..ce50243 100644 --- a/fatcat_scholar/templates/search_macros.html +++ b/fatcat_scholar/templates/search_macros.html @@ -329,7 +329,7 @@ {% endif %} {% if paper.releases|length > 1 %} - {% for release in paper.releases if (release.ident != paper.biblio.release_ident and release.ident != paper.fulltext.release_ident) %} + {% for release in paper.releases if (release.ident != paper.biblio.release_ident and (not paper.fulltext or release.ident != paper.fulltext.release_ident)) %} {% if loop.first %} <h4 class="ui horizontal divider header"> {# <i class="tag icon"></i> #} @@ -386,7 +386,6 @@ <div class="tag-row"> {# ### TAGS #} {# colors to use: olive, brown, grey, pink, red, etc #} - {# TODO: remove doc for ES 7.x-style lack of type #} {# TODO: only show 'json' link if from cluster? #} {% if debug_mode %} <a target="_blank" rel="noopener" href="{{ settings.ELASTICSEARCH_PUBLIC_URL }}/{{ settings.ELASTICSEARCH_QUERY_FULLTEXT_INDEX }}/_doc/{{ paper.key }}"> diff --git a/fatcat_scholar/transform.py b/fatcat_scholar/transform.py index f9616c4..3a7102a 100644 --- a/fatcat_scholar/transform.py +++ b/fatcat_scholar/transform.py @@ -483,7 +483,10 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]: raise NotImplementedError(f"doc_type: {heavy.doc_type}") # TODO: this crude filter should not be necessary once we upgrade to GROBID v0.6+ - if heavy.grobid_fulltext and heavy.grobid_fulltext.get('file_ident') != 'gbbvrg2tpzan5hl3qcsfzh4vfq': + if ( + heavy.grobid_fulltext + and heavy.grobid_fulltext.get("file_ident") != "gbbvrg2tpzan5hl3qcsfzh4vfq" + ): fulltext_release = [ r for r in heavy.releases @@ -603,6 +606,55 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]: ) +def clean_ref_key(key: Optional[str], doi: Optional[str] = None) -> Optional[str]: + if not key: + return None + key = key.strip() + if key and doi and key.startswith(doi): + key = key.replace(doi + "-", "") + key = key.replace(doi, "") + if key.startswith("10.") and "SICI" in key and "-" in key: + subkey = key.split("-")[-1] + if subkey: + key = subkey + if key.startswith("10.") and "_" in key: + subkey = key.split("_")[-1] + if subkey: + key = subkey + if len(key) > 10 and "#" in key: + subkey = key.split("#")[-1] + if subkey: + key = subkey + if len(key) > 10 and "_" in key: + subkey = key.split("_")[-1] + if subkey: + key = subkey + if key and key.startswith("ref-"): + key = key[4:] + if len(key) >= 2 and key[0] in ["/", "_"]: + key = key[1:] + if not key: + return None + return key + + +def test_clean_ref_key() -> None: + test_pairs = [ + ("ref-23", None, "23"), + ("_bib0040", None, "bib0040"), + (" 20170224012016_R15", None, "R15"), + ( + "10.1002/(SICI)1099-1026(199905/06)14:3<195::AID-FFJ807>3.0.CO;2-C-BIB1", + None, + "BIB1", + ), + ("BFnrcardio201557_CR175", None, "CR175"), + ("2019121710443552100_", None, "2019121710443552100_"), + ] + for raw, doi, expected in test_pairs: + assert clean_ref_key(raw, doi=doi) == expected + + def refs_from_grobid(release: ReleaseEntity, tei_dict: dict) -> List[RefStructured]: output = [] for ref in tei_dict.get("citations") or []: @@ -619,6 +671,10 @@ def refs_from_grobid(release: ReleaseEntity, tei_dict: dict) -> List[RefStructur if a.get("name"): assert isinstance(a["name"], str) authors.append(a["name"]) + ref_index = ref.get("index") + if ref_index is not None: + # transform from 0-indexed to 1-indexed + ref_index = ref_index + 1 output.append( RefStructured( biblio=RefBiblio( @@ -636,15 +692,15 @@ def refs_from_grobid(release: ReleaseEntity, tei_dict: dict) -> List[RefStructur pmid=ref.get("pmid"), pmcid=clean_pmcid(ref.get("pmcid")), arxiv_id=ref.get("arxiv_id"), - # isbn13: Optional[str] + isbn=ref.get("isbn"), url=clean_url_conservative(ref.get("url")), ), release_ident=release.ident, work_ident=release.work_id, release_stage=release.release_stage, release_year=release.release_year, - index=ref.get("index"), - key=ref.get("id"), + index=ref_index, + key=clean_ref_key(ref.get("id")), locator=None, # target_release_id ref_source="grobid", @@ -658,14 +714,6 @@ def refs_from_release_refs(release: ReleaseEntity) -> List[RefStructured]: for ref in release.refs: ref_source = "fatcat" - key = ref.key - if key and release.ext_ids.doi and key.startswith(release.ext_ids.doi): - key = key.replace(release.ext_ids.doi, "") - if key and key.startswith("ref-"): - key = key[4:] - if key and key.startswith("b"): - key = key[1:] - if release.extra and release.extra.get("pubmed"): ref_source = "fatcat-pubmed" elif release.extra and release.extra.get("crossref"): @@ -676,6 +724,10 @@ def refs_from_release_refs(release: ReleaseEntity) -> List[RefStructured]: extra = ref.extra or dict() authors = extra.get("authors") or [] authors = [a for a in authors if type(a) == str] + ref_index = None + if ref.index is not None: + # transform from 0-indexed (release.refs) to 1-indexed (fatcat_refs) + ref_index = ref.index + 1 output.append( RefStructured( biblio=RefBiblio( @@ -689,18 +741,19 @@ def refs_from_release_refs(release: ReleaseEntity) -> List[RefStructured]: volume=extra.get("volume"), issue=extra.get("issue"), pages=extra.get("pages") or extra.get("page"), - doi=extra.get("doi"), + doi=clean_doi(extra.get("doi")), pmid=extra.get("pmid"), - pmcid=extra.get("pmcid"), + pmcid=clean_pmcid(extra.get("pmcid")), arxiv_id=extra.get("arxiv_id"), - isbn13=extra.get("isbn13"), + isbn=extra.get("isbn13") or extra.get("isbn"), url=clean_url_conservative(extra.get("url")), ), release_ident=release.ident, work_ident=release.work_id, + release_stage=release.release_stage, release_year=release.release_year, - index=ref.index, - key=key or None, + index=ref_index, + key=clean_ref_key(ref.key, doi=release.ext_ids.doi), locator=ref.locator, target_release_id=ref.target_release_id, ref_source=ref_source, @@ -724,26 +777,41 @@ def refs_from_crossref( authors = [ ref["author"], ] - key = ref.get("key") - if key and key.startswith(record["DOI"]): - key = key.replace(record["DOI"] + "-", "") - key = key.replace(record["DOI"], "") - if key and key.startswith("ref-"): - key = key[4:] + ref_title = ref.get("article-title") ref_container_name = ref.get("journal-title") if not ref_container_name: + ref_container_name = ref.get("container-title") + + # volume-title is often a book title + if not ref_title: + ref_title = ref.get("volume-title") + elif not ref_container_name: ref_container_name = ref.get("volume-title") + + # series-title is a bit weird in Crossref references. it is often + # passed alone and seems to be the article/book title miscategorized. + # other times it is a conference name. + series_title = ref.get("series-title") + if not ref_title: + ref_title = series_title + elif not ref_container_name: + ref_container_name = series_title + + year = ref.get("year") + if year: + year = clean_small_int(year) + else: + year = None date = ref.get("date") - year = None - if date and len(date) >= 4 and date[:4].isdigit(): + if date and not year and len(date) >= 4 and date[:4].isdigit(): year = int(date[:4]) - if year < 1000 or year > 2100: - year = None + if year and (year < 1000 or year > 2100): + year = None output.append( RefStructured( biblio=RefBiblio( unstructured=ref.get("unstructured"), - title=ref.get("article-title"), + title=ref_title, subtitle=ref.get("subtitle"), contrib_raw_names=authors, year=year, @@ -751,15 +819,18 @@ def refs_from_crossref( publisher=ref.get("publisher"), volume=ref.get("volume"), issue=ref.get("issue"), - pages=ref.get("page"), - doi=ref.get("DOI"), + pages=ref.get("first-page"), + version=ref.get("edition"), + doi=clean_doi(ref.get("DOI")), + isbn=ref.get("ISBN"), ), release_ident=release.ident, work_ident=release.work_id, + release_stage=release.release_stage, release_year=release.release_year, - index=i, - key=key or None, - locator=ref.get("first-page"), + index=i + 1, # 1-indexed + key=clean_ref_key(ref.get("key"), doi=record.get("DOI")), + # locator, target_release_id=None, ref_source=ref_source, ) @@ -795,7 +866,10 @@ def refs_from_heavy(heavy: IntermediateBundle) -> Sequence[RefStructured]: fulltext_refs: List[RefStructured] = [] # TODO: this crude filter should not be necessary once we upgrade to GROBID v0.6+ - if heavy.grobid_fulltext and heavy.grobid_fulltext.get('file_ident') != 'gbbvrg2tpzan5hl3qcsfzh4vfq': + if ( + heavy.grobid_fulltext + and heavy.grobid_fulltext.get("file_ident") != "gbbvrg2tpzan5hl3qcsfzh4vfq" + ): fulltext_release = [ r for r in heavy.releases diff --git a/fatcat_scholar/web.py b/fatcat_scholar/web.py index b5af18e..a705e20 100644 --- a/fatcat_scholar/web.py +++ b/fatcat_scholar/web.py @@ -20,6 +20,7 @@ from fastapi.responses import ( RedirectResponse, ) from fastapi.middleware.cors import CORSMiddleware +import fatcat_openapi_client import sentry_sdk from sentry_sdk.integrations.asgi import SentryAsgiMiddleware from starlette_prometheus import metrics, PrometheusMiddleware @@ -182,72 +183,6 @@ def get_work(work_ident: str = Query(..., min_length=20, max_length=20)) -> dict return doc -@api.get( - "/work/{work_ident}/access/wayback/{url:path}", - operation_id="access_redirect_wayback", - include_in_schema=False, -) -def access_redirect_wayback( - url: str, - request: Request, - work_ident: str = Query(..., min_length=20, max_length=20), -) -> Any: - raw_original_url = "/".join(str(request.url).split("/")[7:]) - # the quote() call is necessary because the URL is un-encoded in the path parameter - # see also: https://github.com/encode/starlette/commit/f997938916d20e955478f60406ef9d293236a16d - original_url = urllib.parse.quote(raw_original_url, safe=":/%#?=@[]!$&'()*+,;",) - doc_dict = get_es_scholar_doc(f"work_{work_ident}") - if not doc_dict: - raise HTTPException(status_code=404, detail="work not found") - doc: ScholarDoc = doc_dict["_obj"] - # combine fulltext with all access options - access: List[Any] = [] - if doc.fulltext: - access.append(doc.fulltext) - access.extend(doc.access or []) - for opt in access: - if ( - opt.access_type == "wayback" - and opt.access_url - and "://web.archive.org/web/" in opt.access_url - and opt.access_url.endswith(original_url) - ): - timestamp = opt.access_url.split("/")[4] - if not (len(timestamp) == 14 and timestamp.isdigit()): - continue - access_url = f"https://web.archive.org/web/{timestamp}id_/{original_url}" - return RedirectResponse(access_url, status_code=302) - raise HTTPException(status_code=404, detail="access URL not found") - - -@api.get( - "/work/{work_ident}/access/ia_file/{item}/{file_path:path}", - operation_id="access_redirect_ia_file", - include_in_schema=False, -) -def access_redirect_ia_file( - item: str, - file_path: str, - request: Request, - work_ident: str = Query(..., min_length=20, max_length=20), -) -> Any: - original_path = urllib.parse.quote("/".join(str(request.url).split("/")[8:])) - access_url = f"https://archive.org/download/{item}/{original_path}" - doc_dict = get_es_scholar_doc(f"work_{work_ident}") - if not doc_dict: - raise HTTPException(status_code=404, detail="work not found") - doc: ScholarDoc = doc_dict["_obj"] - # combine fulltext with all access options - access: List[Any] = [] - if doc.fulltext: - access.append(doc.fulltext) - access.extend(doc.access or []) - for opt in access: - if opt.access_type == "ia_file" and opt.access_url == access_url: - return RedirectResponse(access_url, status_code=302) - raise HTTPException(status_code=404, detail="access URL not found") - - web = APIRouter() @@ -413,6 +348,165 @@ def web_work( ) +def access_redirect_fallback( + request: Request, + work_ident: str, + original_url: Optional[str] = None, + archiveorg_path: Optional[str] = None, +) -> Any: + """ + The purpose of this helper is to catch access redirects which would + otherwise return a 404, and "try harder" to find a redirect. + """ + # lookup against the live fatcat API, instead of scholar ES index + api_conf = fatcat_openapi_client.Configuration() + api_conf.host = settings.FATCAT_API_HOST + api_client = fatcat_openapi_client.DefaultApi( + fatcat_openapi_client.ApiClient(api_conf) + ) + + # fetch list of releases for this work from current fatcat catalog. note + # that these releases are not expanded (don't include file entities) + try: + # fetch work entity itself to fail fast (true 404) and handle redirects + work_entity = api_client.get_work(work_ident) + logger.warning( + f"access_redirect_fallback: work_{work_ident} state={work_entity.state} redirect={work_entity.redirect}" + ) + if work_entity.redirect: + work_ident = work_entity.redirect + partial_releases = api_client.get_work_releases( + ident=work_ident, hide="abstracts,references", + ) + except fatcat_openapi_client.ApiException as ae: + raise HTTPException( + status_code=ae.status, + detail=f"Fatcat API call failed for work_{work_ident}", + ) + + # for each release, check for any archive.org access option with the given context + for partial in partial_releases: + release = api_client.get_release( + partial.ident, + expand="files", + # TODO: expand="files,filesets,webcaptures", + hide="abstracts,references", + ) + if not release.files: + continue + for fe in release.files: + for url_pair in fe.urls: + access_url = url_pair.url + if ( + original_url + and "://web.archive.org/web/" in access_url + and access_url.endswith(original_url) + ): + # TODO: test/verify this + timestamp = access_url.split("/")[4] + # if not (len(timestamp) == 14 and timestamp.isdigit()): + # continue + replay_url = ( + f"https://web.archive.org/web/{timestamp}id_/{original_url}" + ) + return RedirectResponse(replay_url, status_code=302) + elif ( + archiveorg_path + and "://archive.org/" in access_url + and archiveorg_path in access_url + ): + return RedirectResponse(access_url, status_code=302) + + # give up and show an error page + lang = LangPrefix(request) + return i18n_templates[lang.code].TemplateResponse( + "access_404.html", + { + "request": request, + "locale": lang.code, + "lang_prefix": lang.prefix, + "work_ident": work_ident, + "original_url": original_url, + "archiveorg_path": archiveorg_path, + }, + status_code=404, + ) + + +@web.get( + "/work/{work_ident}/access/wayback/{url:path}", + operation_id="access_redirect_wayback", + include_in_schema=False, +) +def access_redirect_wayback( + url: str, + request: Request, + work_ident: str = Query(..., min_length=20, max_length=20), +) -> Any: + raw_original_url = "/".join(str(request.url).split("/")[7:]) + # the quote() call is necessary because the URL is un-encoded in the path parameter + # see also: https://github.com/encode/starlette/commit/f997938916d20e955478f60406ef9d293236a16d + original_url = urllib.parse.quote(raw_original_url, safe=":/%#?=@[]!$&'()*+,;",) + doc_dict = get_es_scholar_doc(f"work_{work_ident}") + if not doc_dict: + return access_redirect_fallback( + request, work_ident=work_ident, original_url=original_url + ) + doc: ScholarDoc = doc_dict["_obj"] + # combine fulltext with all access options + access: List[Any] = [] + if doc.fulltext: + access.append(doc.fulltext) + access.extend(doc.access or []) + for opt in access: + if ( + opt.access_type == "wayback" + and opt.access_url + and "://web.archive.org/web/" in opt.access_url + and opt.access_url.endswith(original_url) + ): + timestamp = opt.access_url.split("/")[4] + if not (len(timestamp) == 14 and timestamp.isdigit()): + continue + access_url = f"https://web.archive.org/web/{timestamp}id_/{original_url}" + return RedirectResponse(access_url, status_code=302) + return access_redirect_fallback( + request, work_ident=work_ident, original_url=original_url + ) + + +@web.get( + "/work/{work_ident}/access/ia_file/{item}/{file_path:path}", + operation_id="access_redirect_ia_file", + include_in_schema=False, +) +def access_redirect_ia_file( + item: str, + file_path: str, + request: Request, + work_ident: str = Query(..., min_length=20, max_length=20), +) -> Any: + original_path = urllib.parse.quote("/".join(str(request.url).split("/")[8:])) + access_url = f"https://archive.org/download/{item}/{original_path}" + doc_dict = get_es_scholar_doc(f"work_{work_ident}") + if not doc_dict: + return access_redirect_fallback( + request, work_ident=work_ident, archiveorg_path=f"/{item}/{original_path}" + ) + doc: ScholarDoc = doc_dict["_obj"] + # combine fulltext with all access options + access: List[Any] = [] + if doc.fulltext: + access.append(doc.fulltext) + access.extend(doc.access or []) + for opt in access: + if opt.access_type == "ia_file" and opt.access_url == access_url: + return RedirectResponse(access_url, status_code=302) + return access_redirect_fallback( + request, work_ident=work_ident, archiveorg_path=f"/{item}/{original_path}" + ) + + app = FastAPI( title="Fatcat Scholar", description="Fulltext search interface for scholarly web content in the Fatcat catalog. An Internet Archive project.", |