From 5bbb493b94a63d66151a53837aa66f0d986df497 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 24 Jul 2020 20:02:05 -0700 Subject: first iteration of flexible search coverage --- python/fatcat_web/routes.py | 29 ++++++++ python/fatcat_web/search.py | 36 ++++++++++ python/fatcat_web/templates/container_view.html | 2 +- .../templates/container_view_coverage.html | 2 +- python/fatcat_web/templates/coverage_search.html | 81 ++++++++++++++++++++++ 5 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 python/fatcat_web/templates/coverage_search.html diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 45f6b0b6..a741112f 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -732,6 +732,35 @@ def container_search(): return render_template('container_search.html', query=query, es_error=fse), fse.status_code return render_template('container_search.html', query=query, found=found) +@app.route('/coverage/search', methods=['GET', 'POST']) +def coverage_search(): + + if 'q' not in request.args.keys(): + return render_template( + 'coverage_search.html', + query=ReleaseQuery(), + coverage_stats=None, + coverage_type_preservation=None, + year_histogram_svg=None, + ) + + query = ReleaseQuery.from_args(request.args) + coverage_stats = get_elastic_search_coverage(query) + if coverage_stats['total'] > 1: + year_histogram = get_elastic_preservation_by_year(query) + year_histogram_svg = preservation_by_year_histogram(year_histogram).render_data_uri() + coverage_type_preservation = get_elastic_preservation_by_type(query) + else: + year_histogram_svg = None + coverage_type_preservation = None + return render_template( + 'coverage_search.html', + query=query, + coverage_stats=coverage_stats, + coverage_type_preservation=coverage_type_preservation, + year_histogram_svg=year_histogram_svg, + ) + def get_changelog_stats(): stats = {} latest_changelog = api.get_changelog(limit=1)[0] diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 7ae7e73b..3ba6fdb2 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -366,6 +366,42 @@ def get_elastic_entity_stats() -> dict: return stats +def get_elastic_search_coverage(query: ReleaseQuery) -> dict: + + search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) + search = search.query( + "query_string", + query=query.q, + default_operator="AND", + analyze_wildcard=True, + allow_leading_wildcard=False, + lenient=True, + fields=["biblio"], + ) + search.aggs.bucket( + 'preservation', + 'terms', + field='preservation', + missing='_unknown', + ) + + search = search[:0] + + search = search.params(request_cache=True) + resp = wrap_es_execution(search) + + preservation_bucket = agg_to_dict(resp.aggregations.preservation) + preservation_bucket['total'] = resp.hits.total + for k in ('bright', 'dark', 'shadows_only', 'none'): + if not k in preservation_bucket: + preservation_bucket[k] = 0 + stats = { + 'total': resp.hits.total, + 'preservation': preservation_bucket, + } + + return stats + def get_elastic_container_stats(ident, issnl=None): """ Returns dict: diff --git a/python/fatcat_web/templates/container_view.html b/python/fatcat_web/templates/container_view.html index 785ad1ee..8b4ddeb9 100644 --- a/python/fatcat_web/templates/container_view.html +++ b/python/fatcat_web/templates/container_view.html @@ -86,7 +86,7 @@ {% else %} {{ type_row }} {% endif %} - {{ container._stats.release_type[type_row] }} + {{ "{:,}".format(container._stats.release_type[type_row]) }} {% endfor %} diff --git a/python/fatcat_web/templates/container_view_coverage.html b/python/fatcat_web/templates/container_view_coverage.html index fd173cd4..5ec31e73 100644 --- a/python/fatcat_web/templates/container_view_coverage.html +++ b/python/fatcat_web/templates/container_view_coverage.html @@ -57,7 +57,7 @@ {% for type_row in container._type_preservation %} {{ type_row.release_type }} - {{ type_row.total }} + {{ "{:,}".format(type_row.total) }} {{ entity_macros.preservation_bar(type_row) }} {% endfor %} diff --git a/python/fatcat_web/templates/coverage_search.html b/python/fatcat_web/templates/coverage_search.html new file mode 100644 index 00000000..c730ef9d --- /dev/null +++ b/python/fatcat_web/templates/coverage_search.html @@ -0,0 +1,81 @@ +{% import "entity_macros.html" as entity_macros %} +{% extends "base.html" %} + +{% block title %} +{% if query.q %} + Search: {{ query.q }} +{% else %} + Coverage Search +{% endif %} +{% endblock %} + +{% block fullmain %} + +
+
+

Preservation Coverage Visualizer

+
+
+
+ + +
+
Can also search for releases (eg, individual papers) or containers (eg, journals). +
+
+
+
+ +
+ +{% if coverage_stats != None %} +
+
+
+
+
{{ "{:,}".format(coverage_stats.total) }}
+
Known Releases
+
+ {% if coverage_stats.total >= 1 %} + {{ entity_macros.preservation_bar(coverage_stats.preservation, extra_class="large") }} + {{ entity_macros.preservation_table(coverage_stats.preservation) }} + {% endif %} +
+
+
+{% endif %} + +{% if year_histogram_svg != None %} +

+

Perpetual Access Coverage by Year

+ +
+ +
+ +{% endif %} + +{% if coverage_type_preservation != None %} +

+

Perpetual Access Coverage by Release Type

+ + + + + + {% for type_row in coverage_type_preservation %} + + +
Release Type + Total Count + Coverage +
{{ type_row.release_type }} + {{ "{:,}".format(type_row.total) }} + {{ entity_macros.preservation_bar(type_row) }} + {% endfor %} +
+{% endif %} + +
+{% endblock %} + -- cgit v1.2.3