From d5b24df069fc96d396afbb302633a077e5dbfb39 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 15 Apr 2021 18:34:11 -0700 Subject: first iteration of basic citation inbound/outbound views --- python/fatcat_web/__init__.py | 2 +- python/fatcat_web/ref_routes.py | 50 ++++++++++++ .../templates/release_view_fuzzy_refs.html | 95 ++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 python/fatcat_web/ref_routes.py create mode 100644 python/fatcat_web/templates/release_view_fuzzy_refs.html (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/__init__.py b/python/fatcat_web/__init__.py index 07b4e083..3207bc75 100644 --- a/python/fatcat_web/__init__.py +++ b/python/fatcat_web/__init__.py @@ -76,7 +76,7 @@ app.register_blueprint(mwoauth.bp, url_prefix='/auth/wikipedia') app.es_client = elasticsearch.Elasticsearch(Config.ELASTICSEARCH_BACKEND) -from fatcat_web import routes, editing_routes, auth, cors, forms +from fatcat_web import routes, editing_routes, ref_routes, auth, cors, forms # TODO: blocking on ORCID support in loginpass if Config.ORCID_CLIENT_ID: diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py new file mode 100644 index 00000000..a49813c4 --- /dev/null +++ b/python/fatcat_web/ref_routes.py @@ -0,0 +1,50 @@ +""" +Flask endpoints for reference (citation) endpoints. Eg, listing references +"inbound" and "outbound" from a specific release or work. +""" + +from typing import Optional + +from flask import render_template, abort, redirect, request +from fatcat_openapi_client import * +from fatcat_openapi_client.rest import ApiException + +from fatcat_tools.references import enrich_inbound_refs_fatcat, enrich_outbound_refs_fatcat, get_inbound_refs, get_outbound_refs +from fatcat_web import app, api, auth_api +from fatcat_web.forms import * +from fatcat_web.entity_helpers import * + + +@app.route('/release//refs/inbound', methods=['GET']) +def release_view_refs_inbound(ident): + + # lookup release ident, ensure it exists + try: + release = api.get_release(ident) + except ApiException as ae: + abort(ae.status) + + offset = request.args.get('offset', '0') + offset = max(0, int(offset)) if offset.isnumeric() else 0 + + hits = get_inbound_refs(release_ident=ident, es_client=app.es_client, offset=offset, limit=30) + enriched_refs = enrich_inbound_refs_fatcat(hits.result_refs, fatcat_api_client=api, expand="container,files,webcaptures") + + return render_template('release_view_fuzzy_refs.html', direction="inbound", entity=release, hits=hits, enriched_refs=enriched_refs), 200 + +@app.route('/release//refs/outbound', methods=['GET']) +def release_view_refs_outbound(ident): + + # lookup release ident, ensure it exists + try: + release = api.get_release(ident) + except ApiException as ae: + abort(ae.status) + + offset = request.args.get('offset', '0') + offset = max(0, int(offset)) if offset.isnumeric() else 0 + + hits = get_outbound_refs(release_ident=ident, es_client=app.es_client, offset=offset, limit=30) + enriched_refs = enrich_outbound_refs_fatcat(hits.result_refs, fatcat_api_client=api, expand="container,files,webcaptures") + + return render_template('release_view_fuzzy_refs.html', direction="outbound", entity=release, hits=hits, enriched_refs=enriched_refs), 200 diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html new file mode 100644 index 00000000..bc1fa171 --- /dev/null +++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html @@ -0,0 +1,95 @@ +{% set release = entity %} +{% set entity_view = "references" %} +{% set entity_type = "release" %} +{% import "entity_macros.html" as entity_macros %} +{% extends "entity_base.html" %} + +{% block entity_main %} + +{% if direction == "inbound" %} +

Inbound Matched References

+ Other releases citing this one +{% elif direction == "outbound" %} +

Outbound Matched References

+ This release citing other releases +{% endif %} + +

Found {{ hits.count_total }} references in {{ hits.query_wall_time_ms }}ms. +{% if hits.count_total != hits.count_returned %} + Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} + {% if hits.offset + hits.limit < hits.count_total %} +  next... + {% endif %} +{% endif %} + + + +{% for ref in enriched_refs %} + {% set release = ref.release %} + +
+ {% if direction == "outbound" %} + {% if ref.ref.ref_key %} + [{{ ref.ref.ref_key }}] + {% endif %} + {% endif %} +
{{ ref.ref.match_status }} +
{{ ref.ref.match_provenance }} + 		+ {{ release.title }} + {% if release.release_type not in ["article-journal", "conference-paper"] %} + [{{ release.release_type or "unknown-type" }}] + {% endif %} +
+ {% for contrib in release.contribs[:5] %} + {% if contrib.creator %} + {{ contrib.creator.display_name }} + {% else %} + {{ contrib.raw_name }} + {% endif %} + {% if not loop.last %}, {% endif %} + {% endfor %} + {% if release.contribs | length > 5 %}(+ more) {%endif %} +
+ {% if release.release_year %}{{ release.release_year }}  {% endif %} + {% if release.container %} + {{ release.container.name }} + {% elif release.extra and release.extra.container_name %} + {{ release.extra.container_name }} + {% endif %} + {% if release.release_stage != "published" %} +  {{ release.release_stage or "unpublished" }} + {% endif %} + +
+ {% if release.version %} + version:{{ release.release_year }}  + {% endif %} + {% if release.number %} + number:{{ release.number }}  + {% endif %} + {% if release.ext_ids.doi %} + doi:{{ release.ext_ids.doi }}  + {% endif %} + {# TODO: links #} + {% if release.ext_ids.arxiv %} + arXiv:{{ release.ext_ids.arxiv }}  + {% endif %} + {% if release.ext_ids.pmcid %} + pmcid:{{ release.ext_ids.pmcid }}  + {% endif %} + {% if release.ext_ids.pmid %} + pmid:{{ release.ext_ids.pmid }}  + {% endif %} + {% if release.ext_ids.dblp %} + dblp:{{ release.ext_ids.dblp }}  + {% endif %} + 		+ {% if ref.access %} + {{ ref.access[0].access_type.name }} + {% endif %} +{% endfor %} +
+ +{% endblock %} + -- cgit v1.2.3 From 7186a379f335dd2731d5db79ab85abf3506cee88 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 15 Apr 2021 23:25:00 -0700 Subject: web: template macro to display release entry summary --- python/fatcat_web/templates/entity_macros.html | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 50f45753..94770afb 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -387,3 +387,55 @@ yellow {%- endmacro %} + +{# this is useful for things like showing lists of releases in tables #} +{% macro release_summary(release) %} +{{ release.title }} + {% if release.release_type not in ["article-journal", "conference-paper"] %} + [{{ release.release_type or "unknown-type" }}] + {% endif %} +
+ {% for contrib in release.contribs[:5] %} + {% if contrib.creator %} + {{ contrib.creator.display_name }} + {% else %} + {{ contrib.raw_name }} + {% endif %} + {% if not loop.last %}, {% endif %} + {% endfor %} + {% if release.contribs | length > 5 %}(+ more) {%endif %} +
+ {% if release.release_year %}{{ release.release_year }}  {% endif %} + {% if release.container %} + {{ release.container.name }} + {% elif release.extra and release.extra.container_name %} + {{ release.extra.container_name }} + {% endif %} + {% if release.release_stage != "published" %} +  {{ release.release_stage or "unpublished" }} + {% endif %} + +
+ {% if release.version %} + version:{{ release.release_year }}  + {% endif %} + {% if release.number %} + number:{{ release.number }}  + {% endif %} + {% if release.ext_ids.doi %} + doi:{{ release.ext_ids.doi }}  + {% endif %} + {# TODO: links #} + {% if release.ext_ids.arxiv %} + arXiv:{{ release.ext_ids.arxiv }}  + {% endif %} + {% if release.ext_ids.pmcid %} + pmcid:{{ release.ext_ids.pmcid }}  + {% endif %} + {% if release.ext_ids.pmid %} + pmid:{{ release.ext_ids.pmid }}  + {% endif %} + {% if release.ext_ids.dblp %} + dblp:{{ release.ext_ids.dblp }}  + {% endif %} +{% endmacro %} -- cgit v1.2.3 From 314aba35d06eb80be0c5ffc068774bb9bca38e76 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 15 Apr 2021 23:31:07 -0700 Subject: web: initial implementation of fuzzy citation parsing and matching tool --- python/fatcat_web/forms.py | 41 +++++++++++ python/fatcat_web/ref_routes.py | 46 +++++++++++++ python/fatcat_web/templates/reference_match.html | 86 ++++++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 python/fatcat_web/templates/reference_match.html (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py index 1c9fb199..19176a59 100644 --- a/python/fatcat_web/forms.py +++ b/python/fatcat_web/forms.py @@ -482,3 +482,44 @@ class EntityTomlForm(EntityEditForm): etf.toml.data = entity_to_toml(entity, pop_fields=pop_fields) return etf + +class ReferenceMatchForm(FlaskForm): + + submit_type = SelectField('submit_type', + [validators.DataRequired()], + choices=['parse', 'match']) + + raw_citation = TextAreaField("Citation String", render_kw={'rows':'3'}) + + title = StringField("Title") + journal = StringField("Journal or Conference") + first_author = StringField("First Author") + #year = IntegerField('Year Released', + # [validators.Optional(True), valid_year]) + year = StringField("Year Released") + volume = StringField("Volume") + issue = StringField("Issue") + pages = StringField("Pages") + + @staticmethod + def from_grobid_parse(parse_dict, raw_citation): + """ + Initializes form from GROBID extraction + """ + rmf = ReferenceMatchForm() + rmf.raw_citation.data = raw_citation + + direct_fields = ['title', 'journal', 'volume', 'issue', 'pages'] + for k in direct_fields: + if parse_dict.get(k): + a = getattr(rmf, k) + a.data = parse_dict[k] + + date = parse_dict.get('date') + if date and len(date) >= 4 and date[0:4].isdigit(): + rmf.year.data = int(date[0:4]) + + if parse_dict.get('authors'): + rmf.first_author.data = parse_dict['authors'][0].get('name') + + return rmf diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py index a49813c4..dc39299f 100644 --- a/python/fatcat_web/ref_routes.py +++ b/python/fatcat_web/ref_routes.py @@ -8,8 +8,11 @@ from typing import Optional from flask import render_template, abort, redirect, request from fatcat_openapi_client import * from fatcat_openapi_client.rest import ApiException +from fuzzycat.grobid_unstructured import grobid_api_process_citation, transform_grobid_ref_xml, grobid_ref_to_release +from fuzzycat.simple import close_fuzzy_biblio_matches, close_fuzzy_release_matches from fatcat_tools.references import enrich_inbound_refs_fatcat, enrich_outbound_refs_fatcat, get_inbound_refs, get_outbound_refs +from fatcat_tools.transforms.access import release_access_options from fatcat_web import app, api, auth_api from fatcat_web.forms import * from fatcat_web.entity_helpers import * @@ -48,3 +51,46 @@ def release_view_refs_outbound(ident): enriched_refs = enrich_outbound_refs_fatcat(hits.result_refs, fatcat_api_client=api, expand="container,files,webcaptures") return render_template('release_view_fuzzy_refs.html', direction="outbound", entity=release, hits=hits, enriched_refs=enriched_refs), 200 + +@app.route('/reference/match', methods=['GET', 'POST']) +def reference_match(): + + form = ReferenceMatchForm() + grobid_status = None + grobid_dict = None + + if form.is_submitted(): + if form.validate_on_submit(): + if form.submit_type.data == 'parse': + resp_xml = grobid_api_process_citation(form.raw_citation.data) + if not resp_xml: + grobid_status = "failed" + return render_template('reference_match.html', form=form, grobid_status=grobid_status), 400 + grobid_dict = transform_grobid_ref_xml(resp_xml) + if not grobid_dict: + grobid_status = "empty" + return render_template('reference_match.html', form=form, grobid_status=grobid_status), 200 + #print(grobid_dict) + release_stub = grobid_ref_to_release(grobid_dict) + # remove empty values from GROBID parsed dict + grobid_dict = {k: v for k, v in grobid_dict.items() if v is not None} + form = ReferenceMatchForm.from_grobid_parse(grobid_dict, form.raw_citation.data) + grobid_status = "success" + matches = close_fuzzy_release_matches(es_client=app.es_client, release=release_stub, match_limit=10) or [] + elif form.submit_type.data == 'match': + matches = close_fuzzy_biblio_matches(es_client=app.es_client, biblio=form.data, match_limit=10) or [] + else: + raise NotImplementedError() + + for m in matches: + # expand releases more completely + m.release = api.get_release(m.release.ident, expand="container,files,filesets,webcaptures", hide="abstract,refs") + # hack in access options + m.access_options = release_access_options(m.release) + + return render_template('reference_match.html', form=form, grobid_dict=grobid_dict, grobid_status=grobid_status, matches=matches), 200 + + elif form.errors: + return render_template('reference_match.html', form=form), 400 + + return render_template('reference_match.html', form=form), 200 diff --git a/python/fatcat_web/templates/reference_match.html b/python/fatcat_web/templates/reference_match.html new file mode 100644 index 00000000..042b0607 --- /dev/null +++ b/python/fatcat_web/templates/reference_match.html @@ -0,0 +1,86 @@ +{% extends "base.html" %} +{% import "entity_macros.html" as entity_macros %} +{% import "edit_macros.html" as edit_macros %} + +{% block body %} + +

Reference Fuzzy Match Tool

+ +
+ + +

Parse Citation

+ +

Enter a citation string here and we will try to parse it (using GROBID) + into a structured format, then match against the catalog. + + {{ edit_macros.form_field_basic(form.raw_citation) }} + + + +
+ {% if grobid_status == "success" and grobid_dict %} +

+
Parsed successfully! See match results below
+ {{ entity_macros.extra_metadata(grobid_dict) }} +
+ {% endif %} + +
+
+

Fuzzy Match Metadata

+ +

Enter whatever bibliographic metadata fields you know, and we will try to + match to catalog entries. + +

NOTE: if you already know a persistent identifier (like a DOI), you + should use the lookup tool instead. + + {{ edit_macros.form_field_inline(form.title) }} + {{ edit_macros.form_field_inline(form.first_author) }} + +
+

+ {{ edit_macros.form_field_basic(form.year) }} + {{ edit_macros.form_field_basic(form.journal) }} +
+
+ {{ edit_macros.form_field_basic(form.volume) }} + {{ edit_macros.form_field_basic(form.issue) }} + {{ edit_macros.form_field_basic(form.pages) }} +
+ + +
+ +
+ +{% if matches is defined %} +
+
+

Match Results

+ + + {% for match in matches %} + +
+
{{ match.status.name }} +
{{ match.reason.name }} + 		+ {{ entity_macros.release_summary(match.release) }} + + {% if match.access_options %} + {{ match.access_options[0].access_type.name }} + {% endif %} + {% endfor %} +
+ {% if not matches %} +

None! + {% endif %} +{% endif %} + +{% endblock %} -- cgit v1.2.3 From 570074b514259bf6345c376faea8128f279bd0b4 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 15 Apr 2021 23:48:06 -0700 Subject: web: inbound/outbound refs as links (temporarily); change URL names --- python/fatcat_web/ref_routes.py | 4 ++-- python/fatcat_web/templates/entity_base.html | 4 ++++ python/fatcat_web/templates/release_view_fuzzy_refs.html | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py index dc39299f..bd8ae550 100644 --- a/python/fatcat_web/ref_routes.py +++ b/python/fatcat_web/ref_routes.py @@ -18,7 +18,7 @@ from fatcat_web.forms import * from fatcat_web.entity_helpers import * -@app.route('/release//refs/inbound', methods=['GET']) +@app.route('/release//inbound-refs', methods=['GET']) def release_view_refs_inbound(ident): # lookup release ident, ensure it exists @@ -35,7 +35,7 @@ def release_view_refs_inbound(ident): return render_template('release_view_fuzzy_refs.html', direction="inbound", entity=release, hits=hits, enriched_refs=enriched_refs), 200 -@app.route('/release//refs/outbound', methods=['GET']) +@app.route('/release//outbound-refs', methods=['GET']) def release_view_refs_outbound(ident): # lookup release ident, ensure it exists diff --git a/python/fatcat_web/templates/entity_base.html b/python/fatcat_web/templates/entity_base.html index 36280f5d..c23dbef2 100644 --- a/python/fatcat_web/templates/entity_base.html +++ b/python/fatcat_web/templates/entity_base.html @@ -86,6 +86,10 @@ {% elif entity_type == "release" and entity.state != 'deleted' %} {{ entity_tab("contribs", "Authors", "/contribs", entity._authors|count ) }} {{ entity_tab("references", "References", "/references", entity.refs|count) }} + {% if entity.state == 'active' %} + {{ entity_tab("inbound-refs", "Inbound", "/inbound-refs") }} + {{ entity_tab("outbound-refs", "Outbound", "/outbound-refs") }} + {% endif %} {% endif %} {{ entity_tab("metadata", "Metadata", "/metadata") }} diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html index bc1fa171..9ceb6060 100644 --- a/python/fatcat_web/templates/release_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html @@ -1,5 +1,5 @@ {% set release = entity %} -{% set entity_view = "references" %} +{% set entity_view = "{{ direction }}-refs" %} {% set entity_type = "release" %} {% import "entity_macros.html" as entity_macros %} {% extends "entity_base.html" %} -- cgit v1.2.3 From c2395869ff7860bb2c7f080fd6c097e299ea58bf Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 23 Jun 2021 19:49:36 -0700 Subject: fixes for newer ref index --- python/fatcat_tools/references.py | 2 +- .../templates/release_view_fuzzy_refs.html | 59 ++++------------------ 2 files changed, 11 insertions(+), 50 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_tools/references.py b/python/fatcat_tools/references.py index 7e1f4f71..976967d4 100644 --- a/python/fatcat_tools/references.py +++ b/python/fatcat_tools/references.py @@ -68,7 +68,7 @@ class BiblioRef(BaseModel): # skipped: target_wikipedia_article # crossref, pubmed, grobid, etc - match_provenance: str + match_provenance: Optional[str] # strong, weak, etc match_status: Optional[str] # TODO: "match_strength"? diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html index 9ceb6060..ee39d15b 100644 --- a/python/fatcat_web/templates/release_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html @@ -33,56 +33,17 @@ {% endif %} {% endif %}
{{ ref.ref.match_status }} -
{{ ref.ref.match_provenance }} + {% if ref.ref.match_provenance %} +
{{ ref.ref.match_provenance }} + {% endif %} - {{ release.title }} - {% if release.release_type not in ["article-journal", "conference-paper"] %} - [{{ release.release_type or "unknown-type" }}] - {% endif %} -
- {% for contrib in release.contribs[:5] %} - {% if contrib.creator %} - {{ contrib.creator.display_name }} - {% else %} - {{ contrib.raw_name }} - {% endif %} - {% if not loop.last %}, {% endif %} - {% endfor %} - {% if release.contribs | length > 5 %}(+ more) {%endif %} -
- {% if release.release_year %}{{ release.release_year }}  {% endif %} - {% if release.container %} - {{ release.container.name }} - {% elif release.extra and release.extra.container_name %} - {{ release.extra.container_name }} - {% endif %} - {% if release.release_stage != "published" %} -  {{ release.release_stage or "unpublished" }} - {% endif %} - -
- {% if release.version %} - version:{{ release.release_year }}  - {% endif %} - {% if release.number %} - number:{{ release.number }}  - {% endif %} - {% if release.ext_ids.doi %} - doi:{{ release.ext_ids.doi }}  - {% endif %} - {# TODO: links #} - {% if release.ext_ids.arxiv %} - arXiv:{{ release.ext_ids.arxiv }}  - {% endif %} - {% if release.ext_ids.pmcid %} - pmcid:{{ release.ext_ids.pmcid }}  - {% endif %} - {% if release.ext_ids.pmid %} - pmid:{{ release.ext_ids.pmid }}  - {% endif %} - {% if release.ext_ids.dblp %} - dblp:{{ release.ext_ids.dblp }}  - {% endif %} + {% if release %} + {% entity_macros.release_summary(release) %} + {% elif ref.ref.target_unstructured %} + {{ ref.ref.target_unstructured }} + {% else %} + blank + {% endif %} {% if ref.access %} {{ ref.access[0].access_type.name }} -- cgit v1.2.3 From 61ed521cc40c1ee76692e9c4054e89fa63320600 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 24 Jun 2021 18:45:08 -0700 Subject: improvements to fuzzy refs view - fixes to release summary macro - show tab counts correctly by re-using generic entity get helper - table styling; 'prev' link - openlibrary access links - parse-and-match button for unmatched+unstructured refs --- python/fatcat_web/ref_routes.py | 12 +-- python/fatcat_web/templates/entity_macros.html | 17 ++-- .../templates/release_view_fuzzy_refs.html | 93 +++++++++++++++------- 3 files changed, 75 insertions(+), 47 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py index bd8ae550..e08aaf15 100644 --- a/python/fatcat_web/ref_routes.py +++ b/python/fatcat_web/ref_routes.py @@ -21,11 +21,7 @@ from fatcat_web.entity_helpers import * @app.route('/release//inbound-refs', methods=['GET']) def release_view_refs_inbound(ident): - # lookup release ident, ensure it exists - try: - release = api.get_release(ident) - except ApiException as ae: - abort(ae.status) + release = generic_get_entity("release", ident) offset = request.args.get('offset', '0') offset = max(0, int(offset)) if offset.isnumeric() else 0 @@ -38,11 +34,7 @@ def release_view_refs_inbound(ident): @app.route('/release//outbound-refs', methods=['GET']) def release_view_refs_outbound(ident): - # lookup release ident, ensure it exists - try: - release = api.get_release(ident) - except ApiException as ae: - abort(ae.status) + release = generic_get_entity("release", ident) offset = request.args.get('offset', '0') offset = max(0, int(offset)) if offset.isnumeric() else 0 diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 94770afb..24d1b6d0 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -394,30 +394,29 @@ yellow {% if release.release_type not in ["article-journal", "conference-paper"] %} [{{ release.release_type or "unknown-type" }}] {% endif %} -
+ {% if release.contribs %}
{% endif %} {% for contrib in release.contribs[:5] %} {% if contrib.creator %} - {{ contrib.creator.display_name }} + {{ contrib.creator.display_name }} {% else %} {{ contrib.raw_name }} - {% endif %} - {% if not loop.last %}, {% endif %} + {%- endif %} + {%- if not loop.last %}, {% endif %} {% endfor %} - {% if release.contribs | length > 5 %}(+ more) {%endif %} -
+ {% if release.contribs | length > 5 %} (+ more) {%endif %} + {% if release.release_year or release.container or (release.extra and release.extra.container_name) %}
{% endif %} {% if release.release_year %}{{ release.release_year }}  {% endif %} {% if release.container %} - {{ release.container.name }} + {{ release.container.name }} {% elif release.extra and release.extra.container_name %} {{ release.extra.container_name }} {% endif %} {% if release.release_stage != "published" %}  {{ release.release_stage or "unpublished" }} {% endif %} -
{% if release.version %} - version:{{ release.release_year }}  + version:{{ release.version }}  {% endif %} {% if release.number %} number:{{ release.number }}  diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html index ee39d15b..7b286fd3 100644 --- a/python/fatcat_web/templates/release_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html @@ -4,53 +4,90 @@ {% import "entity_macros.html" as entity_macros %} {% extends "entity_base.html" %} +{% macro pagination_row(hits) %} + + {% if hits.offset %} + « prev   + {% endif %} + Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} of {{ hits.count_total}} references (in {{ hits.query_wall_time_ms }}ms) + {% if hits.count_total != hits.count_returned and hits.offset + hits.limit < hits.count_total %} +  next » + {% endif %} +{% endmacro %} + {% block entity_main %} {% if direction == "inbound" %} -

Inbound Matched References

- Other releases citing this one +

Referenced By

+ Citations to this release by other works. {% elif direction == "outbound" %} -

Outbound Matched References

- This release citing other releases -{% endif %} - -

Found {{ hits.count_total }} references in {{ hits.query_wall_time_ms }}ms. -{% if hits.count_total != hits.count_returned %} - Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} - {% if hits.offset + hits.limit < hits.count_total %} -  next... - {% endif %} +

References

+ NOTE: currently batch computed and may include additional references sources, or be missing recent changes, compared to entity reference list. {% endif %} - +{% if enriched_refs %} +
+ + -{% for ref in enriched_refs %} - {% set release = ref.release %} - +{% if hits.count_total != hits.count_returned %} + + +{% endif %}
+ {{ pagination_row(hits) }} +
+{% for row in enriched_refs %} + {% set release = row.release %} +
+ {# TODO: ref_locator? #} {% if direction == "outbound" %} - {% if ref.ref.ref_key %} - [{{ ref.ref.ref_key }}] + {% if row.ref.ref_key %} + [{{ row.ref.ref_key }}]
{% endif %} {% endif %} -
{{ ref.ref.match_status }} - {% if ref.ref.match_provenance %} -
{{ ref.ref.match_provenance }} + {{ row.ref.match_status }}
+ {% if row.ref.match_provenance %} + via {{ row.ref.match_provenance }} {% endif %} 		{% if release %} - {% entity_macros.release_summary(release) %} - {% elif ref.ref.target_unstructured %} - {{ ref.ref.target_unstructured }} + {{ entity_macros.release_summary(release) }} + {% elif row.ref.target_unstructured %} + {{ row.ref.target_unstructured }} {% else %} blank {% endif %} - - {% if ref.access %} - {{ ref.access[0].access_type.name }} - {% endif %} + + {% if row.access %} + {% for access in row.access %} + {{ access.access_type.name }}
+ {% endfor %} + {% elif row.ref.target_unstructured %} +
+ + + +
+ {% endif %} + + {# TODO: include these as access options instead #} + {% if row.ref.target_openlibrary_work %} + openlibrary.org + {% endif %} + {% if row.ref.target_url %} + web +
wayback (?) + {% endif %} {% endfor %}
+ {{ pagination_row(hits) }} +
+{% else %} +

None found +{% endif %} {% endblock %} -- cgit v1.2.3 From 7498fb076d0b60a9021f7174f0a5c4ead38c151a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 24 Jun 2021 18:46:45 -0700 Subject: match UI: improve form layout --- python/fatcat_web/templates/reference_match.html | 29 +++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/reference_match.html b/python/fatcat_web/templates/reference_match.html index 042b0607..ae6a239c 100644 --- a/python/fatcat_web/templates/reference_match.html +++ b/python/fatcat_web/templates/reference_match.html @@ -38,19 +38,22 @@

NOTE: if you already know a persistent identifier (like a DOI), you should use the lookup tool instead. - {{ edit_macros.form_field_inline(form.title) }} - {{ edit_macros.form_field_inline(form.first_author) }} - -
-

- {{ edit_macros.form_field_basic(form.year) }} - {{ edit_macros.form_field_basic(form.journal) }} -
-
- {{ edit_macros.form_field_basic(form.volume) }} - {{ edit_macros.form_field_basic(form.issue) }} - {{ edit_macros.form_field_basic(form.pages) }} -
+
+
+ {{ edit_macros.form_field_basic(form.title) }} +
+
+ {{ edit_macros.form_field_basic(form.first_author) }} +
+
+ {{ edit_macros.form_field_basic(form.journal) }} +
+
+ {{ edit_macros.form_field_basic(form.year) }} + {{ edit_macros.form_field_basic(form.volume) }} + {{ edit_macros.form_field_basic(form.issue) }} + {{ edit_macros.form_field_basic(form.pages) }} +
+ +
+ -
{% if grobid_status == "success" and grobid_dict %}
-
Parsed successfully! See match results below
+
Parsed Citation String
{{ entity_macros.extra_metadata(grobid_dict) }} +

See below for fuzzy match results

{% endif %} -
-
-

Fuzzy Match Metadata

+
+

Fuzzy Match Metadata

-

Enter whatever bibliographic metadata fields you know, and we will try to - match to catalog entries. +

Enter whatever bibliographic metadata fields you know, and we will try to + match to catalog entries. -

NOTE: if you already know a persistent identifier (like a DOI), you - should use the lookup tool instead. +

NOTE: if you already know a persistent identifier (like a DOI), you + should use the lookup tool instead. -
-

- {{ edit_macros.form_field_basic(form.title) }} -
-
- {{ edit_macros.form_field_basic(form.first_author) }} -
-
- {{ edit_macros.form_field_basic(form.journal) }} -
-
- {{ edit_macros.form_field_basic(form.year) }} - {{ edit_macros.form_field_basic(form.volume) }} - {{ edit_macros.form_field_basic(form.issue) }} - {{ edit_macros.form_field_basic(form.pages) }} -
+
+
+ {{ edit_macros.form_field_basic(form.title) }} +
+
+ {{ edit_macros.form_field_basic(form.first_author) }} +
+
+ {{ edit_macros.form_field_basic(form.journal) }} +
+
+ {{ edit_macros.form_field_basic(form.year) }} + {{ edit_macros.form_field_basic(form.volume) }} + {{ edit_macros.form_field_basic(form.issue) }} + {{ edit_macros.form_field_basic(form.pages) }} +
- -
+ +
+
{% if matches is defined %} -
-
-

Match Results

+

Matched Releases

+ + {% if not matches %} +

No matches found + {% endif %} + {% for match in matches %} @@ -81,9 +87,7 @@ {% endfor %}
- {% if not matches %} -

None! - {% endif %} + {% endif %} {% endblock %} -- cgit v1.2.3 From fa2ba60834cf3cb3edea05af3c1830e6fc0d5bcc Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 27 Jul 2021 18:48:14 -0700 Subject: refs: several small improvements to web UI --- python/fatcat_web/templates/entity_macros.html | 11 +++- .../templates/openlibrary_view_fuzzy_refs.html | 6 +- python/fatcat_web/templates/refs_macros.html | 71 ++++++++++++++++------ .../templates/release_view_fuzzy_refs.html | 12 ++-- .../templates/wikipedia_view_fuzzy_refs.html | 6 +- 5 files changed, 71 insertions(+), 35 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 37bb2d90..562b99d3 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -391,7 +391,7 @@ yellow {# this is useful for things like showing lists of releases in tables #} {% macro release_summary(release) %} {{ release.title }} - {% if release.release_type not in ["article-journal", "conference-paper"] %} + {% if release.release_type not in ["article-journal", "paper-conference"] %} [{{ release.release_type or "unknown-type" }}] {% endif %} {% if release.contribs %}
{% endif %} @@ -405,7 +405,14 @@ yellow {% endfor %} {% if release.contribs | length > 8 %} (+ more) {%endif %} {% if release.release_year or release.container or (release.extra and release.extra.container_name) %}
{% endif %} - {% if release.release_year %}{{ release.release_year }}  {% endif %} + {% if release.release_year %} + {% if release.release_date %} + {{ release.release_year }} + {% else %} + {{ release.release_year }} + {% endif %} +   + {% endif %} {% if release.container %} {{ release.container.name }} {% elif release.extra and release.extra.container_name %} diff --git a/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html index 161a7b50..21bf76f2 100644 --- a/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/openlibrary_view_fuzzy_refs.html @@ -19,11 +19,7 @@ Refernces from this book to other entities. {% endif %} -{% if hits.result_refs %} - {{ refs_macros.refs_table(hits, direction) }} -{% else %} -

None found -{% endif %} +{{ refs_macros.refs_table(hits, direction) }} {% endblock %} diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html index ba4d18ad..3db47064 100644 --- a/python/fatcat_web/templates/refs_macros.html +++ b/python/fatcat_web/templates/refs_macros.html @@ -1,37 +1,67 @@ {% import "entity_macros.html" as entity_macros %} -{% macro pagination_row(hits, direction) %} - - {% if hits.offset %} +{% macro pagination_row(hits, with_links=False) %} + {% if with_links and hits.offset %} « prev   {% endif %} - Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} of {{ hits.count_total}} references (in {{ hits.query_wall_time_ms }}ms) - {% if hits.count_total != hits.count_returned and hits.offset + hits.limit < hits.count_total %} + {% if hits.count_returned == 0 %} + Showing 0 references + {% else %} + Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} of {{ hits.count_total}} references + {% endif %} + {% if with_links and hits.count_total != hits.count_returned and hits.offset + hits.limit < hits.count_total %}  next » {% endif %} {% endmacro %} {% macro refs_table(hits, direction) %} +

+
+ Fuzzy reference matching is a work in progress! +
+ Read more about quality, completeness, and caveats in the fatcat guide. +
+{% if hits.count_total == 0 %} + + -{% if hits.count_total != hits.count_returned %} - - -{% endif %} + +
- {{ pagination_row(hits) }} + {{ pagination_row(hits, with_links=False) }} + (in {{ hits.query_wall_time_ms }}ms)
+
+ + No References Found +
+{% endif %} {% for row in hits.result_refs %} {% set release = row.release %} -
+
{# TODO: ref_locator? #} {% if direction == "out" %} {% if row.ref.ref_key %} [{{ row.ref.ref_key }}]
{% endif %} {% endif %} - {{ row.ref.match_status }}
- {% if row.ref.match_provenance %} - via {{ row.ref.match_provenance }} + + {% if row.ref.match_status == "exact" %} + {% set match_icon = "linkify" %} + {% elif row.ref.match_status == "unmatched" %} + {% set match_icon = "question circle outline" %} + {% else %} + {% set match_icon = "magic" %} {% endif %} +
+ {% if row.ref.match_provenance and row.ref.match_provenance == "fatcat-pubmed" %} + {# this is a common case and making the column render wide #} + via pubmed + {% elif row.ref.match_provenance %} + via {{ row.ref.match_provenance }}
+ {% endif %} + 		{% if release %} {{ entity_macros.release_summary(release) }} @@ -62,7 +92,7 @@ {% if row.access %} {% for access in row.access %} - + {%- if access.access_type.name == "wayback" %} web.archive.org {%- elif access.access_type.name == "ia_file" -%} @@ -83,18 +113,23 @@ {% endif %} {% endfor %}
- {{ pagination_row(hits) }} -
+
+ JSON +
+ {% if hits.count_returned != hits.count_total %} +
+ {{ pagination_row(hits, with_links=True) }} +
+ {% endif %} +
{% endmacro %} diff --git a/python/fatcat_web/templates/release_view_fuzzy_refs.html b/python/fatcat_web/templates/release_view_fuzzy_refs.html index ffca0bc9..8cba4f4e 100644 --- a/python/fatcat_web/templates/release_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/release_view_fuzzy_refs.html @@ -13,13 +13,15 @@ {% elif direction == "out" %}

References

NOTE: currently batch computed and may include additional references sources, or be missing recent changes, compared to entity reference list. -{% endif %} -{% if hits.result_refs %} - {{ refs_macros.refs_table(hits, direction) }} -{% else %} -

None found + {% if hits.count_total == 0 and release.refs %} +

+

No fuzzy references found, but there are {{ release.refs|count }} legacy references +

+ {% endif %} {% endif %} +{{ refs_macros.refs_table(hits, direction) }} + {% endblock %} diff --git a/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html b/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html index 5b53d692..3e1453c1 100644 --- a/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html +++ b/python/fatcat_web/templates/wikipedia_view_fuzzy_refs.html @@ -17,11 +17,7 @@ Refernces from wikipedia article to other entities. {% endif %} -{% if hits.result_refs %} - {{ refs_macros.refs_table(hits, direction) }} -{% else %} -

None found -{% endif %} +{{ refs_macros.refs_table(hits, direction) }} {% endblock %} -- cgit v1.2.3 From f3481c02bd7a50d9073902dba07fe265eecb93db Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 27 Jul 2021 18:57:09 -0700 Subject: refs: lint fixes --- python/fatcat_tools/references.py | 1 + python/fatcat_web/ref_routes.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_tools/references.py b/python/fatcat_tools/references.py index 508cf19d..496a46e1 100644 --- a/python/fatcat_tools/references.py +++ b/python/fatcat_tools/references.py @@ -99,6 +99,7 @@ class EnrichedBiblioRef(BaseModel): access: List[AccessOption] @validator('release') + @classmethod def check_release(cls, v): if v is not None and not isinstance(v, ReleaseEntity): raise ValueError("expected a ReleaseEntity") diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py index 88ac0744..d4219012 100644 --- a/python/fatcat_web/ref_routes.py +++ b/python/fatcat_web/ref_routes.py @@ -3,7 +3,7 @@ Flask endpoints for reference (citation) endpoints. Eg, listing references "inbound" and "outbound" from a specific release or work. """ -from flask import render_template, request, jsonify, Response +from flask import render_template, request, Response from fatcat_openapi_client import * from fuzzycat.grobid_unstructured import grobid_api_process_citation, transform_grobid_ref_xml, grobid_ref_to_release from fuzzycat.simple import close_fuzzy_biblio_matches, close_fuzzy_release_matches @@ -154,7 +154,7 @@ def release_view_refs_inbound_json(ident): @app.route('/openlibrary/OLW/refs-in.json', methods=['GET', 'OPTIONS']) @crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) -def openlibrary_view_refs_inbound_json(ident): +def openlibrary_view_refs_inbound_json(id_num): openlibrary_id = f"OL{id_num}W" hits = _refs_web("in", openlibrary_id=openlibrary_id) return Response(hits.json(exclude_unset=True), mimetype="application/json") -- cgit v1.2.3 From f29da9adb50a37cb6aad4e435fc09a5d682cbe6c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 27 Jul 2021 18:57:25 -0700 Subject: refs: revert fatcat-pubmed -> pubmed truncation This was just going to be confusing --- python/fatcat_web/templates/refs_macros.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html index 3db47064..8b6a5dc3 100644 --- a/python/fatcat_web/templates/refs_macros.html +++ b/python/fatcat_web/templates/refs_macros.html @@ -55,10 +55,7 @@ {% set match_icon = "magic" %} {% endif %}
- {% if row.ref.match_provenance and row.ref.match_provenance == "fatcat-pubmed" %} - {# this is a common case and making the column render wide #} - via pubmed - {% elif row.ref.match_provenance %} + {% if row.ref.match_provenance %} via {{ row.ref.match_provenance }}
{% endif %} -- cgit v1.2.3 From ed56037d929d50abab707ee5eb9f583789a8ac7a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 27 Jul 2021 19:21:49 -0700 Subject: refs: fix typo preventing CSL from rendering in refs output --- python/fatcat_web/templates/refs_macros.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html index 8b6a5dc3..3095ce54 100644 --- a/python/fatcat_web/templates/refs_macros.html +++ b/python/fatcat_web/templates/refs_macros.html @@ -81,7 +81,7 @@ openlibrary:{{ row.ref.target_openlibrary_work }}  [cited-by]  {% endif %} - {% elif direction == "in" and row.ref.target_csl %} + {% elif direction == "out" and row.ref.target_csl %} {{ entity_macros.csl_summary(row.ref.target_csl) }} {% else %} blank -- cgit v1.2.3 From 37b6e99eec3cbc668d6b51ed9e57b93f9a114d2a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 3 Aug 2021 18:15:10 -0700 Subject: refs: web UI tweaks for iterated CSL schema --- python/fatcat_web/templates/entity_macros.html | 22 +++++++++++++++++++--- python/tests/files/elastic_refs_out_release.json | 10 +++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 562b99d3..6b565f69 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -457,15 +457,31 @@ yellow {% if csl.title and csl.author %}
{% endif %} {% if csl.author %} {% for author in csl.author[:8] %} - {# TODO: other name variants? #} - {{ author.name }} + {% if author.literal %} + {{ author.literal }} + {% elif author.raw_name %} + {{ author.raw_name }} + {% elif author.family and author.given %} + {{ author.given }} {{ author.family }} + {% elif author.family %} + {{ author.family }} + {% elif author.name %} + {# DEPRECATED: was used by refs code path for a while. Delete in, eg, year 2022 #} + {{ author.name }} + {% endif %} {%- if not loop.last %}, {% endif %} {% endfor %} {% if csl.author | length > 8 %} (+ more) {%endif %} {% endif %} {% if csl.issued or csl["container-title"] %}
{% endif %} - {% if csl.issued and csl.issued.raw %}{{ csl.issued.raw }}  {% endif %} + {% if csl.issued and csl.issued is mapping %} + {% if csl.issued['date-parts'] %} + {{ csl.issued['date-parts'][0][0] }}   + {% elif csl.issued.raw %} + {{ csl.issued.raw }}   + {% endif %} + {% endif %} {% if csl["container-title"] %} {{ csl["container-title"] }} {% endif %} diff --git a/python/tests/files/elastic_refs_out_release.json b/python/tests/files/elastic_refs_out_release.json index 5e25d80f..5a45acee 100644 --- a/python/tests/files/elastic_refs_out_release.json +++ b/python/tests/files/elastic_refs_out_release.json @@ -152,11 +152,13 @@ "accessed": {}, "author": [ { - "name": "Alatawi Eman" + "raw_name": "Alatawi Eman" } ], "container-title": "Symbolic Execution with Invariant Inlay: Evaluating the Potential. In 2018 25th Australasian Software Engineering Conference, ASWEC 2018.", - "issued": {} + "issued": { + "date-parts": [[2019]] + } } }, "sort": [ @@ -185,7 +187,9 @@ } ], "container-title": "Symbolic Path-Oriented Test Data Generation for Floating-Point Programs. In Sixth IEEE International Conference on Software Testing, Verification and Validation, ICST", - "issued": {} + "issued": { + "raw": "2000" + } } }, "sort": [ -- cgit v1.2.3 From 4338ec346381647f079a464092e45c609894fe11 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 4 Aug 2021 12:11:20 -0700 Subject: refs web: correct URL to refs section of guide --- python/fatcat_web/templates/refs_macros.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html index 3095ce54..4ccca7a5 100644 --- a/python/fatcat_web/templates/refs_macros.html +++ b/python/fatcat_web/templates/refs_macros.html @@ -19,7 +19,7 @@

Fuzzy reference matching is a work in progress!
- Read more about quality, completeness, and caveats in the fatcat guide. + Read more about quality, completeness, and caveats in the fatcat guide. -- cgit v1.2.3 From 56e4ce2d8347cdfedd492d54fde080772f3d8725 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 6 Aug 2021 11:58:10 -0700 Subject: refs: format (commas) large refs hit counts --- python/fatcat_web/templates/refs_macros.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/fatcat_web') diff --git a/python/fatcat_web/templates/refs_macros.html b/python/fatcat_web/templates/refs_macros.html index 4ccca7a5..47ea2dcf 100644 --- a/python/fatcat_web/templates/refs_macros.html +++ b/python/fatcat_web/templates/refs_macros.html @@ -7,7 +7,7 @@ {% if hits.count_returned == 0 %} Showing 0 references {% else %} - Showing {{ hits.offset + 1 }} - {{ hits.offset + hits.count_returned }} of {{ hits.count_total}} references + Showing {{ "{:,}".format(hits.offset + 1) }} - {{ "{:,}".format(hits.offset + hits.count_returned) }} of {{ "{:,}".format(hits.count_total) }} references {% endif %} {% if with_links and hits.count_total != hits.count_returned and hits.offset + hits.limit < hits.count_total %}  next » -- cgit v1.2.3