Merge branch 'bnewbold-refs-apis'

8 files changed, 427 insertions, 14 deletions

diff --git a/python/fatcat_web/templates/entity_base.html b/python/fatcat_web/templates/entity_base.html
index 36280f5d..52acd70a 100644
--- a/python/fatcat_web/templates/entity_base.html
+++ b/python/fatcat_web/templates/entity_base.html
@@ -85,7 +85,10 @@
               {{ entity_tab("coverage", "Coverage", "/coverage") }}
             {% elif entity_type == "release" and entity.state != 'deleted' %}
               {{ entity_tab("contribs", "Authors", "/contribs", entity._authors|count ) }}
-              {{ entity_tab("references", "References", "/references", entity.refs|count) }}
+              {% if  entity.state == 'active' %}
+                {{ entity_tab("refs-out", "References", "/refs-out") }}
+                {{ entity_tab("refs-in", "Cited By", "/refs-in") }}
+              {% endif %}
             {% endif %}
             {{ entity_tab("metadata", "Metadata", "/metadata") }}
           </div>
diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html
index 50f45753..6b565f69 100644
--- a/python/fatcat_web/templates/entity_macros.html
+++ b/python/fatcat_web/templates/entity_macros.html
@@ -387,3 +387,112 @@ yellow
   </table>
 
 {%- endmacro %}
+
+{# this is useful for things like showing lists of releases in tables #}
+{% macro release_summary(release) %}
+  <b><a href="/release/{{ release.ident }}">{{ release.title }}</a></b>
+  {% if release.release_type not in ["article-journal", "paper-conference"] %}
+    <b>[{{ release.release_type or "unknown-type" }}]</b>
+  {% endif %}
+  {% if release.contribs %}<br>{% endif %}
+  {% for contrib in release.contribs[:8] %}
+    {% if contrib.creator %}
+      <a href="/contib/{{ contrib.creator.ident }}" style="color: black;">{{ contrib.creator.display_name }}</a>
+    {% else %}
+      {{ contrib.raw_name }}
+    {%- endif %}
+    {%- if not loop.last %}, {% endif %}
+  {% endfor %}
+  {% if release.contribs | length > 8 %} <i>(+ more)</i> {%endif %}
+  {% if release.release_year or release.container or (release.extra and release.extra.container_name) %}<br>{% endif %}
+  {% if release.release_year %}
+    {% if release.release_date %}
+      <span title="{{ release.release_date }}">{{ release.release_year }}</span>
+    {% else %}
+      {{ release.release_year }}
+    {% endif %}
+    &nbsp;
+  {% endif %}
+  {% if release.container %}
+    <a href="/container/{{ release.container.ident }}" style="color: black;"><i>{{ release.container.name }}</i></a>
+  {% elif release.extra and release.extra.container_name %}
+    <i>{{ release.extra.container_name }}</i>
+  {% endif %}
+
+  {% if release.release_stage == "submitted" %}
+    &nbsp;<b style="color: brown; text-transform: uppercase;">pre-print</b>
+  {% elif release.release_stage and release.release_stage != "published" %}
+    &nbsp;<b style="color: brown; text-transform: uppercase;">{{ release.release_stage }} version</b>
+  {% elif not release.release_stage %}
+    &nbsp;<b style="color: brown; text-transform: uppercase;">unpublished</b>
+  {% endif %}
+<br>
+  {% if release.version %}
+    <span style="color:green">version:{{ release.version }}</span>&nbsp;
+  {% endif %}
+  {% if release.number %}
+    <span style="color:green">number:{{ release.number }}</span>&nbsp;
+  {% endif %}
+  {% if release.ext_ids.doi %}
+    <a href="https://doi.org/{{ release.ext_ids.doi }}" style="color:green;">doi:{{ release.ext_ids.doi }}</a>&nbsp;
+  {% endif %}
+  {# TODO: links #}
+  {% if release.ext_ids.arxiv %}
+    <a href="#" style="color:green;">arXiv:{{ release.ext_ids.arxiv }}</a>&nbsp;
+  {% endif %}
+  {% if release.ext_ids.pmcid %}
+    <a href="#" style="color:green;">pmcid:{{ release.ext_ids.pmcid }}</a>&nbsp;
+  {% endif %}
+  {% if release.ext_ids.pmid %}
+    <a href="#" style="color:green;">pmid:{{ release.ext_ids.pmid }}</a>&nbsp;
+  {% endif %}
+  {% if release.ext_ids.dblp %}
+    <a href="#" style="color:green;">dblp:{{ release.ext_ids.dblp }}</a>&nbsp;
+  {% endif %}
+{% endmacro %}
+
+{# similar to the release_summary above, but for CSL-JSON #}
+{% macro csl_summary(csl) %}
+  <b>{{ csl.title }}</b>
+  {% if csl.title and csl.author %}<br>{% endif %}
+  {% if csl.author %}
+    {% for author in csl.author[:8] %}
+      {% if author.literal %}
+        {{ author.literal }}
+      {% elif author.raw_name %}
+        {{ author.raw_name }}
+      {% elif author.family and author.given %}
+        {{ author.given }} {{ author.family }}
+      {% elif author.family %}
+        {{ author.family }}
+      {% elif author.name %}
+        {# DEPRECATED: was used by refs code path for a while. Delete in, eg, year 2022 #}
+        {{ author.name }}
+      {% endif %}
+      {%- if not loop.last %}, {% endif %}
+    {% endfor %}
+    {% if csl.author | length > 8 %} <i>(+ more)</i> {%endif %}
+  {% endif %}
+
+  {% if csl.issued or csl["container-title"] %}<br>{% endif %}
+  {% if csl.issued and csl.issued is mapping %}
+    {% if csl.issued['date-parts'] %}
+      {{ csl.issued['date-parts'][0][0] }} &nbsp;
+    {% elif csl.issued.raw %}
+      {{ csl.issued.raw }} &nbsp;
+    {% endif %}
+  {% endif %}
+  {% if csl["container-title"] %}
+    <i>{{ csl["container-title"] }}</i>
+  {% endif %}
+  <br>
+  {% if csl.volume %}
+    <span style="color:green">volume:{{ csl.volume}}</span>&nbsp;
+  {% endif %}
+  {% if csl.DOI %}
+    <a href="https://doi.org/{{ csl.DOI }}" style="color:green;">doi:{{ csl.DOI }}</a>&nbsp;
+  {% endif %}
+  {% if csl.URL %}
+    <a href="{{ csl.URL }}" style="color:green;">url:{{ csl.URL }}</a>&nbsp;
+  {% endif %}
+{% endmacro %}
diff --git a/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html
new file mode 100644
index 00000000..21bf76f2
--- /dev/null
+++ b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+{% import "refs_macros.html" as refs_macros %}
+
+{% block title %}Open Library Refs{% endblock %}
+
+{% block fullbody %}
+<h1 class="ui header">
+  {% if hits.result_refs and hits.result_refs[0].ref.target_unstructured %}
+    <i>{{ hits.result_refs[0].ref.target_unstructured }}</i>
+  {% endif %}
+  <span class="sub header"><a href="https://openlibrary.org/works/{{ openlibrary_id }}"><code>https://openlibrary.org/works/{{ openlibrary_id }}</code></a></span>
+</h1>
+
+{% if direction == "in" %}
+  <h3>Cited By</h3>
+  <p>This page lists references to this book from other works (eg, journal articles).
+{% elif direction == "out" %}
+  <h3>References</h3>
+  <i>Refernces from this book to other entities.</i>
+{% endif %}
+
+{{ refs_macros.refs_table(hits, direction) }}
+
+{% endblock %}
+
diff --git a/python/fatcat_web/templates/reference_match.html b/python/fatcat_web/templates/reference_match.html
new file mode 100644
index 00000000..f2335f52
--- /dev/null
+++ b/python/fatcat_web/templates/reference_match.html
@@ -0,0 +1,93 @@
+{% extends "base.html" %}
+{% import "entity_macros.html" as entity_macros %}
+{% import "edit_macros.html" as edit_macros %}
+
+{% block body %}
+
+<h1>Reference Fuzzy Match Tool</h1>
+
+<form class="ui form" id="reference_match" method="POST" action="/reference/match">
+  <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+
+  <div class="ui segment">
+    <h3>Parse Citation</h3>
+
+    <p>Enter a citation string here and we will try to parse it (using GROBID)
+    into a structured format, then match against the catalog.
+
+    {{ edit_macros.form_field_basic(form.raw_citation) }}
+
+    <button class="ui primary submit button right floated" type="submit" name="submit_type" value="parse">
+      Parse
+    </button>
+    <br clear="all">
+  </div>
+
+  {% if grobid_status == "success" and grobid_dict %}
+    <div class="ui positive message">
+      <div class="header">Parsed Citation String</div>
+      {{ entity_macros.extra_metadata(grobid_dict) }}
+      <p><i>See below for fuzzy match results</i>
+    </div>
+  {% endif %}
+
+  <div class="ui segment">
+    <h3>Fuzzy Match Metadata</h3>
+
+    <p>Enter whatever bibliographic metadata fields you know, and we will try to
+    match to catalog entries.
+
+    <p><b>NOTE:</b> if you already know a persistent identifier (like a DOI), you
+    should use the <a href="/release/lookup">lookup tool</a> instead.
+
+    <br>
+    <div class="ui equal width fields">
+      {{ edit_macros.form_field_basic(form.title) }}
+    </div>
+    <div class="ui equal width fields">
+      {{ edit_macros.form_field_basic(form.first_author) }}
+    </div>
+    <div class="ui equal width fields">
+      {{ edit_macros.form_field_basic(form.journal) }}
+    </div>
+    <div class="ui equal width fields">
+      {{ edit_macros.form_field_basic(form.year) }}
+      {{ edit_macros.form_field_basic(form.volume) }}
+      {{ edit_macros.form_field_basic(form.issue) }}
+      {{ edit_macros.form_field_basic(form.pages) }}
+    </div>
+
+    <button class="ui primary submit button right floated" type="submit" name="submit_type" value="match">
+      Match
+    </button>
+    <br clear="all">
+  </div>
+
+</form>
+
+{% if matches is defined %}
+  <h3>Matched Releases</h3>
+
+  {% if not matches %}
+    <p><i>No matches found</i>
+  {% endif %}
+
+  <table class="ui very basic celled table">
+  <tbody>
+  {% for match in matches %}
+    <tr><td class="collapsing center aligned">
+          <br><b>{{ match.status.name }}</b>
+          <br>{{ match.reason.name }}
+        <td class="">
+          {{ entity_macros.release_summary(match.release) }}
+        <td class="">
+          {% if match.access_options %}
+            <a href="{{ match.access_options[0].access_url}}" class="ui tiny green active button">{{ match.access_options[0].access_type.name }}</a>
+            {% endif %}
+  {% endfor %}
+  </tbody>
+  </table>
+
+{% endif %}
+
+{% endblock %}
diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html
new file mode 100644
index 00000000..47ea2dcf
--- /dev/null
+++ b/python/fatcat_web/templates/refs_macros.html
@@ -0,0 +1,132 @@
+{% import "entity_macros.html" as entity_macros %}
+
+{% macro pagination_row(hits, with_links=False) %}
+  {% if with_links and hits.offset %}
+    <a href="?offset={{ hits.offset - hits.limit }}">&laquo; prev</a> &nbsp;
+  {% endif %}
+  {% if hits.count_returned == 0 %}
+    Showing 0 references
+  {% else %}
+    Showing {{ "{:,}".format(hits.offset + 1) }} - {{ "{:,}".format(hits.offset + hits.count_returned) }} of {{ "{:,}".format(hits.count_total) }} references
+  {% endif %}
+  {% if with_links and hits.count_total != hits.count_returned and hits.offset + hits.limit < hits.count_total %}
+    &nbsp;<a href="?offset={{ hits.offset + hits.limit }}">next &raquo;</a>
+  {% endif %}
+{% endmacro %}
+
+{% macro refs_table(hits, direction) %}
+<div class="ui warning message">
+  <div class="header">
+    Fuzzy reference matching is a work in progress!
+  </div>
+  Read more about quality, completeness, and caveats <a href="https://guide.fatcat.wiki/reference_graph.html">in the fatcat guide</a>.
+</div>
+
+<table class="ui table">
+<thead>
+  <tr><th colspan="3">
+    {{ pagination_row(hits, with_links=False) }}
+    (in {{ hits.query_wall_time_ms }}ms)
+</thead>
+<tbody>
+{% if hits.count_total == 0 %}
+  <tr><td class="ui placeholder segment">
+    <div class="ui icon header">
+      <i class="unlink icon"></i>
+      No References Found
+    </div>
+{% endif %}
+{% for row in hits.result_refs %}
+  {% set release = row.release %}
+  <tr>
+      <td class="collapsing left aligned top aligned">
+        {# TODO: ref_locator? #}
+        {% if direction == "out" %}
+          {% if row.ref.ref_key %}
+            <code title="index={{ row.ref.ref_index }}">[{{ row.ref.ref_key }}]</code><br>
+          {% endif %}
+        {% endif %}
+
+        {% if row.ref.match_status == "exact" %}
+          {% set match_icon = "linkify" %}
+        {% elif row.ref.match_status == "unmatched" %}
+          {% set match_icon = "question circle outline" %}
+        {% else %}
+          {% set match_icon = "magic" %}
+        {% endif %}
+        <i class="{{ match_icon }} icon" title="{{ row.ref.match_status }} {{ row.ref.match_reason }}"></i><br>
+        {% if row.ref.match_provenance %}
+          via {{ row.ref.match_provenance }}<br>
+        {% endif %}
+
+      <td class="">
+        {% if release %}
+          {{ entity_macros.release_summary(release) }}
+        {% elif direction == "in" and row.ref.source_wikipedia_article %}
+          {% set wiki_lang = row.ref.source_wikipedia_article.split(':')[0] %}
+          {% set wiki_article = ':'.join(row.ref.source_wikipedia_article.split(':')[1:]) %}
+          <b>
+            <a href="https://{{ wiki_lang }}.wikipedia.org/wiki/{{ wiki_article.replace(' ', '_') }}">
+              {{ wiki_article }}
+            </a>
+            [wikipedia]
+          </b>
+          <br>
+          <span style="color:green;">lang:{{ wiki_lang }}</span>&nbsp;
+          <a href="/wikipedia/{{ wiki_lang }}:{{ wiki_article.replace(' ', '_') }}/refs-out" style="color:green;">[references]</a>&nbsp;
+        {% elif direction == "out" and row.ref.target_unstructured %}
+          <code>{{ row.ref.target_unstructured }}</code>
+          {% if row.ref.target_openlibrary_work %}
+            <br>
+            <a href="https://openlibrary.org/{{ row.ref.target_openlibrary_work }}" style="color:green;">openlibrary:{{ row.ref.target_openlibrary_work }}</a>&nbsp;
+            <a href="/openlibrary/{{ row.ref.target_openlibrary_work}}/refs-in" style="color:green;">[cited-by]</a>&nbsp;
+          {% endif %}
+        {% elif direction == "out" and row.ref.target_csl %}
+          {{ entity_macros.csl_summary(row.ref.target_csl) }}
+        {% else %}
+          <i>blank</i>
+        {% endif %}
+      <td class="center aligned">
+        {% if row.access %}
+          {% for access in row.access %}
+            <a href="{{ access.access_url}}" class="ui green label" style="background-color: #2ca048;">
+              {%- if access.access_type.name == "wayback" %}
+                web.archive.org
+              {%- elif access.access_type.name == "ia_file" -%}
+                archive.org
+              {%- else -%}
+                {{ access.access_type.name }}
+              {%- endif -%}
+              {%- if access.mimetype == "application/pdf" %}
+                [PDF]
+              {%- elif access.mimetype == "text/html" %}
+                [HTML]
+              {%- endif -%}
+            </a>
+            <br>
+          {% endfor %}
+        {% elif direction == "out" and row.ref.target_unstructured %}
+          <form class="ui form" id="reference_match" method="POST" action="/reference/match">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+            <input type="hidden" name="raw_citation" value="{{ row.ref.target_unstructured }}">
+            <button class="ui tiny primary submit button" type="submit" name="submit_type" value="parse">
+              re-parse
+            </button>
+          </form>
+        {% endif %}
+{% endfor %}
+</tbody>
+<tfoot>
+  <tr><th colspan="3">
+    <div style="float: right;">
+      <a href="{{ request.path }}.json?{{ request.query_string.decode() }}">JSON</a>
+    </div>
+    {% if hits.count_returned != hits.count_total %}
+      <center>
+        {{ pagination_row(hits, with_links=True) }}
+      </center>
+    {% endif %}
+</tfoot>
+</table>
+{% endmacro %}
+
diff --git a/python/fatcat_web/templates/release_view.html b/python/fatcat_web/templates/release_view.html
index abf7ace0..4652f4a2 100644
--- a/python/fatcat_web/templates/release_view.html
+++ b/python/fatcat_web/templates/release_view.html
@@ -84,9 +84,9 @@
     Published
     {% if release.container.ident %}
       in <a href="/container/{{ release.container.ident }}"><span itemprop="name">{{ release.container.name }}</span></a>
-    {% elif release.extra and release.extra.container_name %}
+    {%- elif release.extra and release.extra.container_name %}
       in <span itemprop="name">{{ release.extra.container_name }}</span>
-    {% endif %}
+    {%- endif %}
   {% else %}
     Released
     {% if release.release_type %}
@@ -95,26 +95,27 @@
     {% if release.container %}
       in <a href="/container/{{ release.container.ident }}"><span itemprop="name">{{ release.container.name }}</span></a>
     {% endif %}
-  {% endif %}
+  {% endif -%}
   {% if release.publisher %}
     by <span itemprop="publisher">{{ release.publisher }}</span>
   {%- endif %}.
 
   <p>
-  {% if release.volume != None %}
-    Volume {{ release.volume }}
-    {%- if release.issue != None %}, {% endif %}
-  {% endif %}
-  {% if release.issue != None %}
-    Issue {{ release.issue}}
+  {% set comma = joiner(", ") %}
+  {% if release.release_year != None %}
+    {{ release.release_year }} &nbsp;
   {% endif %}
-  {% if release.pages != None %}
+  {% if release.volume != None %}
+    {{- comma() }}Volume {{ release.volume -}}
+  {%- endif %}
+  {%- if release.issue != None %}
+    {{- comma() }}Issue {{ release.issue -}}
+  {%- endif %}
+  {%- if release.pages != None %}
+    {{- comma() }}
     {% if release.pages[0].isdigit() %}p{% endif -%}
     {{ release.pages }}
   {% endif %}
-  {% if release.release_year != None %}
-    ({{ release.release_year }})
-  {% endif %}
 </div>
 
 {% if release.abstracts != [] %}
diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html
new file mode 100644
index 00000000..8cba4f4e
--- /dev/null
+++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html
@@ -0,0 +1,27 @@
+{% set release = entity %}
+{% set entity_view = "refs-" + direction %}
+{% set entity_type = "release" %}
+{% import "refs_macros.html" as refs_macros %}
+{% extends "entity_base.html" %}
+
+
+{% block entity_main %}
+
+{% if direction == "in" %}
+  <h3>Cited By</h3>
+  <i>References to this release by other works.</i>
+{% elif direction == "out" %}
+  <h3>References</h3>
+  <i>NOTE: currently batch computed and may include additional references sources, or be missing recent changes, compared to entity reference list.</i>
+
+  {% if hits.count_total == 0 and release.refs %}
+    <div class="ui positive message">
+      <p>No <i>fuzzy</i> references found, but there are <a href="/release/{{ release.ident }}/references">{{ release.refs|count }} legacy references</a>
+    </div>
+  {% endif %}
+{% endif %}
+
+{{ refs_macros.refs_table(hits, direction) }}
+
+{% endblock %}
+
diff --git a/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html b/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html
new file mode 100644
index 00000000..3e1453c1
--- /dev/null
+++ b/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html
@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+{% import "refs_macros.html" as refs_macros %}
+
+{% block title %}Wikipedia Refs{% endblock %}
+
+{% block fullbody %}
+<h1 class="ui header">
+  [{{ wiki_lang }}] {{ wiki_article }}
+  <span class="sub header"><a href="{{ wiki_url }}"><code>{{ wiki_url }}</code></a></span>
+</h1>
+
+{% if direction == "in" %}
+  <h3>Cited By</h3>
+  <p>This page lists references to a wikipedia article, from other works (eg, journal articles).
+{% elif direction == "out" %}
+  <h3>References</h3>
+  <i>Refernces from wikipedia article to other entities.</i>
+{% endif %}
+
+{{ refs_macros.refs_table(hits, direction) }}
+
+{% endblock %}
+