diff options
| -rw-r--r-- | fatcat_scholar/hacks.py | 69 | ||||
| -rw-r--r-- | fatcat_scholar/search.py | 2 | ||||
| -rw-r--r-- | fatcat_scholar/templates/work.html | 8 | ||||
| -rw-r--r-- | fatcat_scholar/web.py | 65 | ||||
| -rw-r--r-- | tests/test_web.py | 44 |
5 files changed, 149 insertions, 39 deletions
diff --git a/fatcat_scholar/hacks.py b/fatcat_scholar/hacks.py index 5cb8572..1b53e01 100644 --- a/fatcat_scholar/hacks.py +++ b/fatcat_scholar/hacks.py @@ -88,3 +88,72 @@ def test_parse_accept_lang() -> None: parse_accept_lang("en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7", ["zh", "en", "de"]) == "en" ) + + +def wayback_direct_url(url: str) -> str: + """ + Re-writes a wayback replay URL to add the 'id_' suffix (or equivalent for direct file access) + """ + if not "://web.archive.org" in url: + return url + segments = url.split("/") + if len(segments) < 6 or not segments[4].isdigit(): + return url + segments[4] += "id_" + return "/".join(segments) + + +def test_wayback_direct_url() -> None: + assert ( + wayback_direct_url("http://fatcat.wiki/thing.pdf") + == "http://fatcat.wiki/thing.pdf" + ) + assert ( + wayback_direct_url("https://web.archive.org/web/*/http://fatcat.wiki/thing.pdf") + == "https://web.archive.org/web/*/http://fatcat.wiki/thing.pdf" + ) + assert ( + wayback_direct_url( + "https://web.archive.org/web/1234/http://fatcat.wiki/thing.pdf" + ) + == "https://web.archive.org/web/1234id_/http://fatcat.wiki/thing.pdf" + ) + + +def make_access_redirect_url(access_type: str, access_url: str) -> str: + if access_type == "wayback" and "://web.archive.org/" in access_url: + segments = access_url.split("/") + dt = segments[4] + original_url = "/".join(segments[5:]) + return f"https://scholar.archive.org/access/wayback/{dt}/{original_url}" + elif access_type == "ia_file" and "://archive.org/download/" in access_url: + suffix = "/".join(access_url.split("/")[4:]) + return f"https://scholar.archive.org/access/ia_file/{suffix}" + else: + return access_url + + +def test_make_access_redirect_url() -> None: + assert ( + make_access_redirect_url( + "wayback", "https://web.archive.org/web/1234/http://fatcat.wiki/thing.pdf" + ) + == "https://scholar.archive.org/access/wayback/1234/http://fatcat.wiki/thing.pdf" + ) + assert ( + make_access_redirect_url( + "wayback", + "https://web.archive.org/web/1234/http://fatcat.wiki/thing.pdf?param=asdf", + ) + == "https://scholar.archive.org/access/wayback/1234/http://fatcat.wiki/thing.pdf?param=asdf" + ) + assert ( + make_access_redirect_url( + "ia_file", "https://archive.org/download/something/file.pdf" + ) + == "https://scholar.archive.org/access/ia_file/something/file.pdf" + ) + assert ( + make_access_redirect_url("blah", "https://mit.edu/file.pdf") + == "https://mit.edu/file.pdf" + ) diff --git a/fatcat_scholar/search.py b/fatcat_scholar/search.py index 2c02a58..b9ede1d 100644 --- a/fatcat_scholar/search.py +++ b/fatcat_scholar/search.py @@ -463,7 +463,7 @@ def get_es_scholar_doc(key: str) -> Optional[dict]: return doc -def lookup_fulltext_pdf(sha1: str) -> Optional[dict]: +def lookup_fulltext_pdf(sha1: str) -> Optional[ScholarFulltext]: """ Fetch a document by fulltext file sha1, returning only the 'fulltext' sub-document. """ diff --git a/fatcat_scholar/templates/work.html b/fatcat_scholar/templates/work.html index 611576b..67c87e0 100644 --- a/fatcat_scholar/templates/work.html +++ b/fatcat_scholar/templates/work.html @@ -30,10 +30,10 @@ {% if work.biblio.doi %} <meta name="citation_doi" content="{{ work.biblio.doi }}"> {% endif %} -{% if work.fulltext.access_url and work.biblio.release_ident == work.fulltext.release_ident and work.fulltext.access_type in ['wayback', 'ia_file'] and work.fulltext.file_mimetype == "application/pdf" and work.fulltext.file_sha1 %} - <!-- PDF access redirect URL, as requested by, eg, scholar.google.com --> - <meta name="citation_pdf_url" content="https://scholar.archive.org/access-redirect/{{ work.fulltext.file_sha1 }}.pdf"> - <!-- Multiple URLs allowed? <meta name="citation_pdf_url" content="{{ work.fulltext.access_url }}"> --> +{% if work.fulltext.access_url and work.biblio.release_ident == work.fulltext.release_ident and work.fulltext.access_type in ['wayback', 'ia_file'] and work.fulltext.file_mimetype in ["application/pdf", None] and work.fulltext.file_sha1 %} + <!-- single PDF access redirect URL --> + <meta name="citation_pdf_url" content="{{ make_access_redirect_url(work.fulltext.access_type, work.fulltext.access_url) }}"> + <!-- direct URL: {{ work.fulltext.access_url | safe }} --> {% endif %} {% endblock %} diff --git a/fatcat_scholar/web.py b/fatcat_scholar/web.py index 3c8b8f3..f2ef331 100644 --- a/fatcat_scholar/web.py +++ b/fatcat_scholar/web.py @@ -5,6 +5,7 @@ So far there are few endpoints, so we just put them all here! """ import logging +import urllib.parse from typing import Optional, Any, List, Dict from pydantic import BaseModel @@ -25,7 +26,12 @@ from starlette_prometheus import metrics, PrometheusMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException from fatcat_scholar.config import settings, GIT_REVISION -from fatcat_scholar.hacks import Jinja2Templates, parse_accept_lang +from fatcat_scholar.hacks import ( + Jinja2Templates, + parse_accept_lang, + wayback_direct_url, + make_access_redirect_url, +) from fatcat_scholar.search import ( process_query, FulltextQuery, @@ -177,42 +183,15 @@ def get_work(work_ident: str = Query(..., min_length=20, max_length=20)) -> dict return doc -def wayback_direct_url(url: str) -> str: - """ - Re-writes a wayback replay URL to add the 'id_' suffix (or equivalent for direct file access) - """ - if not "://web.archive.org" in url: - return url - segments = url.split("/") - if len(segments) < 6 or not segments[4].isdigit(): - return url - segments[4] += "id_" - return "/".join(segments) - - -def test_wayback_direct_url() -> None: - assert ( - wayback_direct_url("http://fatcat.wiki/thing.pdf") - == "http://fatcat.wiki/thing.pdf" - ) - assert ( - wayback_direct_url("https://web.archive.org/web/*/http://fatcat.wiki/thing.pdf") - == "https://web.archive.org/web/*/http://fatcat.wiki/thing.pdf" - ) - assert ( - wayback_direct_url( - "https://web.archive.org/web/1234/http://fatcat.wiki/thing.pdf" - ) - == "https://web.archive.org/web/1234id_/http://fatcat.wiki/thing.pdf" - ) - - @api.get( "/access-redirect/{sha1}.pdf", operation_id="access_redirect_pdf", include_in_schema=False, ) def access_redirect_pdf(sha1: str = Query(..., min_length=40, max_length=40)) -> Any: + """ + NOTE: DEPRECATED + """ fulltext = lookup_fulltext_pdf(sha1) if not fulltext or not fulltext.access_url: raise HTTPException(status_code=404, detail="PDF file not found") @@ -222,6 +201,28 @@ def access_redirect_pdf(sha1: str = Query(..., min_length=40, max_length=40)) -> return RedirectResponse(access_url, status_code=302) +@api.get( + "/access/wayback/{timestamp}/{url:path}", + operation_id="access_redirect_wayback", + include_in_schema=False, +) +def access_redirect_wayback(timestamp: int, url: str, request: Request) -> Any: + original_url = "/".join(str(request.url).split("/")[6:]) + access_url = f"https://web.archive.org/web/{timestamp}id_/{original_url}" + return RedirectResponse(access_url, status_code=302) + + +@api.get( + "/access/ia_file/{item}/{file_path:path}", + operation_id="access_redirect_ia_file", + include_in_schema=False, +) +def access_redirect_ia_file(item: str, file_path: str, request: Request) -> Any: + original_path = urllib.parse.quote("/".join(str(request.url).split("/")[6:])) + access_url = f"https://archive.org/download/{item}/{original_path}" + return RedirectResponse(access_url, status_code=302) + + web = APIRouter() @@ -270,6 +271,7 @@ def load_i18n_templates() -> Any: # pass-through application settings to be available in templates templates.env.globals["settings"] = settings templates.env.globals["babel_numbers"] = babel.numbers + templates.env.globals["make_access_redirect_url"] = make_access_redirect_url d[lang_opt] = templates return d @@ -414,6 +416,7 @@ async def favicon() -> Any: "fatcat_scholar/static/ia-favicon.ico", media_type="image/x-icon" ) + @app.get("/sitemap.xml", include_in_schema=False) async def basic_sitemap() -> Any: return FileResponse( diff --git a/tests/test_web.py b/tests/test_web.py index 6c6632d..fc7ea14 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -102,6 +102,7 @@ def test_basic_search(client: Any, mocker: Any) -> None: rv = client.get("/zh/search?q=blood") assert rv.status_code == 200 + def test_basic_work_landing_page(client: Any, mocker: Any) -> None: with open("tests/files/elastic_fulltext_get.json") as f: @@ -122,7 +123,11 @@ def test_basic_work_landing_page(client: Any, mocker: Any) -> None: rv = client.get("/zh/work/2x5qvct2dnhrbctqa2q2uyut6a") assert rv.status_code == 200 + def test_basic_access_redirect(client: Any, mocker: Any) -> None: + """ + NOTE: DEPRECATED + """ with open("tests/files/elastic_fulltext_search.json") as f: elastic_resp = json.loads(f.read()) @@ -135,9 +140,42 @@ def test_basic_access_redirect(client: Any, mocker: Any) -> None: (200, {}, json.dumps(elastic_resp)), ] - rv = client.get("/access-redirect/f81f84e23c9ba5d364c70f01fa26e645d29c0427.pdf", allow_redirects=False) + rv = client.get( + "/access-redirect/f81f84e23c9ba5d364c70f01fa26e645d29c0427.pdf", + allow_redirects=False, + ) assert rv.status_code == 302 - assert rv.headers['Location'] == "https://web.archive.org/web/20200206164725id_/https://www.federalreserve.gov/econresdata/feds/2015/files/2015118pap.pdf" + assert ( + rv.headers["Location"] + == "https://web.archive.org/web/20200206164725id_/https://www.federalreserve.gov/econresdata/feds/2015/files/2015118pap.pdf" + ) - rv = client.get("/access-redirect/aaaaaaaaaaaaaaaaaaaaaa01fa26e645d29c0427.pdf", allow_redirects=False) + rv = client.get( + "/access-redirect/aaaaaaaaaaaaaaaaaaaaaa01fa26e645d29c0427.pdf", + allow_redirects=False, + ) assert rv.status_code == 404 + + +def test_access_redirects(client: Any, mocker: Any) -> None: + + # tricky "URL encoding in archive.org path" case + rv = client.get( + "/access/ia_file/crossref-pre-1909-scholarly-works/10.1016%252Fs0140-6736%252802%252912493-7.zip/10.1016%252Fs0140-6736%252802%252912928-x.pdf", + allow_redirects=False, + ) + assert rv.status_code == 302 + assert ( + rv.headers["Location"] + == "https://archive.org/download/crossref-pre-1909-scholarly-works/10.1016%252Fs0140-6736%252802%252912493-7.zip/10.1016%252Fs0140-6736%252802%252912928-x.pdf" + ) + + rv = client.get( + "/access/wayback/20170814015956/https://epub.uni-regensburg.de/21901/1/lorenz73.pdf", + allow_redirects=False, + ) + assert rv.status_code == 302 + assert ( + rv.headers["Location"] + == "https://web.archive.org/web/20170814015956id_/https://epub.uni-regensburg.de/21901/1/lorenz73.pdf" + ) |