refs: generalize web endpoints; JSON content negotiation; openlibrary inbound view; etc

4 files changed, 166 insertions, 41 deletions

diff --git a/python/fatcat_tools/references.py b/python/fatcat_tools/references.py
index 73f57e18..81b55f41 100644
--- a/python/fatcat_tools/references.py
+++ b/python/fatcat_tools/references.py
@@ -10,13 +10,14 @@ import datetime
 import argparse
 from typing import Optional, List, Any, Dict, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, validator
 import elasticsearch
 from elasticsearch_dsl import Search
 from fatcat_openapi_client import ReleaseEntity
 
 from fatcat_tools import public_api
 from fatcat_tools.transforms.access import release_access_options, AccessOption
+from fatcat_tools.transforms.entities import entity_to_dict
 
 
 class BiblioRef(BaseModel):
@@ -97,8 +98,17 @@ class EnrichedBiblioRef(BaseModel):
     # TODO: openlibrary work?
     access: List[AccessOption]
 
+    @validator('release')
+    def check_release(cls, v):
+        if v is not None and not isinstance(v, ReleaseEntity):
+            raise ValueError("expected a ReleaseEntity")
+        return v
+
     class Config:
         arbitrary_types_allowed = True
+        json_encoders = {
+            ReleaseEntity: entity_to_dict,
+        }
 
 
 class RefHits(BaseModel):
@@ -110,6 +120,11 @@ class RefHits(BaseModel):
     query_wall_time_ms: int
     result_refs: List[Union[BiblioRef,EnrichedBiblioRef]]
 
+    class Config:
+        json_encoders = {
+            ReleaseEntity: entity_to_dict,
+        }
+
 
 def _execute_ref_query(search: Any, limit: int, offset: Optional[int] = None) -> RefHits:
     """
@@ -268,40 +283,58 @@ def count_inbound_refs(
 def enrich_inbound_refs(refs: List[BiblioRef], fatcat_api_client: Any, hide: Optional[str] = "refs", expand: Optional[str] = "container,files,webcaptures,filesets") -> List[EnrichedBiblioRef]:
     enriched = []
     for ref in refs:
+        release = None
+        access = []
         if ref.source_release_ident:
             release = fatcat_api_client.get_release(ref.source_release_ident, hide=hide, expand=expand)
-            enriched.append(EnrichedBiblioRef(
-                ref=ref,
-                #csl=None,
-                access=release_access_options(release),
-                release=release,
-            ))
-        else:
-            enriched.append(EnrichedBiblioRef(
-                ref=ref,
-                #csl=None,
-                access=[],
-                release=None,
+            access = release_access_options(release)
+        if ref.source_wikipedia_article:
+            wiki_lang = ref.source_wikipedia.split(':')[0]
+            wiki_article = ':'.join(ref.source_wikipedia.split(':')[1:])
+            access.append(AccessOption(
+                access_type="wikipedia",
+                access_url=f"https://{wiki_lang}.wikipedia.org/wiki/{wiki_article}",
+                mimetype=None,
+                size_bytes=None,
+                thumbnail_url=None
             ))
+        enriched.append(EnrichedBiblioRef(
+            ref=ref,
+            access=access,
+            release=release,
+        ))
     return enriched
 
 
 def enrich_outbound_refs(refs: List[BiblioRef], fatcat_api_client: Any, hide: Optional[str] = "refs", expand: Optional[str] = "container,files,webcaptures,filesets") -> List[EnrichedBiblioRef]:
     enriched = []
     for ref in refs:
+        release = None
+        access = []
         if ref.target_release_ident:
             release = fatcat_api_client.get_release(ref.target_release_ident, hide=hide, expand=expand)
-            enriched.append(EnrichedBiblioRef(
-                ref=ref,
-                access=release_access_options(release),
-                release=release,
+            access = release_access_options(release)
+        if ref.target_openlibrary_work:
+            access.append(AccessOption(
+                access_type="openlibrary",
+                access_url=f"https://openlibrary.org/works/{ref.target_openlibrary_work}",
+                mimetype=None,
+                size_bytes=None,
+                thumbnail_url=None
             ))
-        else:
-            enriched.append(EnrichedBiblioRef(
-                ref=ref,
-                access=[],
-                release=None,
+        if ref.target_url and '://web.archive.org/' in ref.target_url:
+            access.append(AccessOption(
+                access_type="wayback",
+                access_url=ref.target_url,
+                mimetype=None,
+                size_bytes=None,
+                thumbnail_url=None
             ))
+        enriched.append(EnrichedBiblioRef(
+            ref=ref,
+            access=access,
+            release=release,
+        ))
     return enriched
 
 
diff --git a/python/fatcat_tools/transforms/access.py b/python/fatcat_tools/transforms/access.py
index add8ff3b..5ed64c7c 100644
--- a/python/fatcat_tools/transforms/access.py
+++ b/python/fatcat_tools/transforms/access.py
@@ -13,6 +13,8 @@ class AccessType(str, Enum):
     ia_file = "ia_file"
     ia_microfilm = "ia_microfilm"
     repository = "repository"
+    openlibrary = "openlibrary"
+    wikipedia = "wikipedia"
 
 class AccessOption(BaseModel):
 
diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py
index 7c3ba5bd..72f115cf 100644
--- a/python/fatcat_web/ref_routes.py
+++ b/python/fatcat_web/ref_routes.py
@@ -3,44 +3,82 @@ Flask endpoints for reference (citation) endpoints. Eg, listing references
 "inbound" and "outbound" from a specific release or work.
 """
 
-from flask import render_template, request
+from flask import render_template, request, jsonify, Response
 from fatcat_openapi_client import *
 from fuzzycat.grobid_unstructured import grobid_api_process_citation, transform_grobid_ref_xml, grobid_ref_to_release
 from fuzzycat.simple import close_fuzzy_biblio_matches, close_fuzzy_release_matches
 
-from fatcat_tools.references import enrich_inbound_refs, enrich_outbound_refs, get_inbound_refs, get_outbound_refs
+from fatcat_tools.references import enrich_inbound_refs, enrich_outbound_refs, get_inbound_refs, get_outbound_refs, RefHits
 from fatcat_tools.transforms.access import release_access_options
 from fatcat_web import app, api
+from fatcat_web.cors import crossdomain
 from fatcat_web.forms import *
 from fatcat_web.entity_helpers import *
 
-
-@app.route('/release/<string(length=26):ident>/refs/in', methods=['GET'])
-def release_view_refs_inbound(ident):
-
-    release = generic_get_entity("release", ident)
-
+def _refs_web(direction, release_ident=None, work_ident=None, openlibrary_id=None) -> RefHits:
     offset = request.args.get('offset', '0')
     offset = max(0, int(offset)) if offset.isnumeric() else 0
+    limit = request.args.get('offset', '30')
+    limit = min(max(0, int(limit)), 100) if limit.isnumeric() else 30
+    if direction == "in":
+        hits = get_inbound_refs(
+            release_ident=release_ident,
+            work_ident=work_ident,
+            openlibrary_work=openlibrary_id,
+            es_client=app.es_client,
+            offset=offset,
+            limit=limit,
+        )
+        hits.result_refs = enrich_inbound_refs(
+            hits.result_refs,
+            fatcat_api_client=api,
+            expand="container,files,webcaptures",
+        )
+    elif direction == "out":
+        hits = get_outbound_refs(
+            release_ident=release_ident,
+            work_ident=work_ident,
+            es_client=app.es_client,
+            offset=offset,
+            limit=limit,
+        )
+        hits.result_refs = enrich_outbound_refs(
+            hits.result_refs,
+            fatcat_api_client=api,
+            expand="container,files,webcaptures",
+        )
+    else:
+        raise ValueError()
+    return hits
+
+
+@app.route('/release/<string(length=26):ident>/refs-in', methods=['GET'])
+def release_view_refs_inbound(ident):
+    if request.accept_mimetypes.best == "application/json":
+        return release_view_refs_inbound_json(ident)
 
-    hits = get_inbound_refs(release_ident=ident, es_client=app.es_client, offset=offset, limit=30)
-    enriched_refs = enrich_inbound_refs(hits.result_refs, fatcat_api_client=api, expand="container,files,webcaptures")
-
-    return render_template('release_view_fuzzy_refs.html', direction="in", entity=release, hits=hits, enriched_refs=enriched_refs), 200
+    release = generic_get_entity("release", ident)
+    hits = _refs_web("in", release_ident=ident)
+    return render_template('release_view_fuzzy_refs.html', direction="in", entity=release, hits=hits), 200
 
 
-@app.route('/release/<string(length=26):ident>/refs/out', methods=['GET'])
+@app.route('/release/<string(length=26):ident>/refs-out', methods=['GET'])
 def release_view_refs_outbound(ident):
+    if request.accept_mimetypes.best == "application/json":
+        return release_view_refs_outbound_json(ident)
 
     release = generic_get_entity("release", ident)
+    hits = _refs_web("out", release_ident=ident)
+    return render_template('release_view_fuzzy_refs.html', direction="out", entity=release, hits=hits), 200
 
-    offset = request.args.get('offset', '0')
-    offset = max(0, int(offset)) if offset.isnumeric() else 0
-
-    hits = get_outbound_refs(release_ident=ident, es_client=app.es_client, offset=offset, limit=30)
-    enriched_refs = enrich_outbound_refs(hits.result_refs, fatcat_api_client=api, expand="container,files,webcaptures")
+@app.route('/openlibrary/OL<int:id_num>W/refs-in', methods=['GET'])
+def openlibrary_view_refs_inbound(id_num):
+    if request.accept_mimetypes.best == "application/json":
+        return openlibrary_view_refs_inbound(id_num)
 
-    return render_template('release_view_fuzzy_refs.html', direction="out", entity=release, hits=hits, enriched_refs=enriched_refs), 200
+    openlibrary_id = f"OL{id_num}W"
+    hits = _refs_web("in", openlibrary_id=openlibrary_id)
+    return render_template('openlibrary_view_fuzzy_refs.html', openlibrary_id=openlibrary_id, direction="in", hits=hits), 200
 
 
 @app.route('/reference/match', methods=['GET', 'POST'])
@@ -85,3 +123,26 @@ def reference_match():
             return render_template('reference_match.html', form=form), 400
 
     return render_template('reference_match.html', form=form), 200
+
+
+### Pseudo-APIs #############################################################
+
+@app.route('/release/<string(length=26):ident>/refs-out.json', methods=['GET', 'OPTIONS'])
+@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type'])
+def release_view_refs_outbound_json(ident):
+    hits = _refs_web("out", release_ident=ident)
+    return Response(hits.json(exclude_unset=True), mimetype="application/json")
+
+
+@app.route('/release/<string(length=26):ident>/refs-in.json', methods=['GET', 'OPTIONS'])
+@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type'])
+def release_view_refs_inbound_json(ident):
+    hits = _refs_web("in", release_ident=ident)
+    return Response(hits.json(exclude_unset=True), mimetype="application/json")
+
+@app.route('/openlibrary/OL<int:id_num>W/refs-in', methods=['GET', 'OPTIONS'])
+@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type'])
+def openlibrary_view_refs_inbound_json(ident):
+    openlibrary_id = f"OL{id_num}W"
+    hits = _refs_web("in", openlibrary_id=openlibrary_id)
+    return Response(hits.json(exclude_unset=True), mimetype="application/json")
diff --git a/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html
new file mode 100644
index 00000000..161a7b50
--- /dev/null
+++ b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html
@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+{% import "refs_macros.html" as refs_macros %}
+
+{% block title %}Open Library Refs{% endblock %}
+
+{% block fullbody %}
+<h1 class="ui header">
+  {% if hits.result_refs and hits.result_refs[0].ref.target_unstructured %}
+    <i>{{ hits.result_refs[0].ref.target_unstructured }}</i>
+  {% endif %}
+  <span class="sub header"><a href="https://openlibrary.org/works/{{ openlibrary_id }}"><code>https://openlibrary.org/works/{{ openlibrary_id }}</code></a></span>
+</h1>
+
+{% if direction == "in" %}
+  <h3>Cited By</h3>
+  <p>This page lists references to this book from other works (eg, journal articles).
+{% elif direction == "out" %}
+  <h3>References</h3>
+  <i>Refernces from this book to other entities.</i>
+{% endif %}
+
+{% if hits.result_refs %}
+  {{ refs_macros.refs_table(hits, direction) }}
+{% else %}
+  <br><br><p><b>None found</b>
+{% endif %}
+
+{% endblock %}
+