From 6244c06abf8488fff87b30cb0a8433592f1f5d24 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 8 Feb 2022 14:35:39 -0800 Subject: container search: iterate on SERP page (including stats) --- python/fatcat_web/templates/entity_macros.html | 63 ++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) (limited to 'python/fatcat_web/templates/entity_macros.html') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 9b419c41..b801f3c5 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -267,23 +267,70 @@ {% endmacro %} -{% macro container_search_result_row(entity) -%} +{% macro container_search_result_row(entity, show_stats=False) -%}
-

+ {% if show_stats %} +
+ {% if entity.releases_total %} + ~{{ "{:,}".format(entity.releases_total) }} releases + {% elif entity.releases_total == 0 %} + (not indexed) + {% endif %} + {% if entity.releases_total %} + {{ preservation_bar({'bright': entity.preservation_bright, 'dark': entity.preservation_dark, 'none': entity.preservation_none, 'total': entity.releases_total}) }} + {% endif %} +
+ {% endif %} +

{{ entity['name'] }} - {% if entity.is_oa %}{% endif %}

+ {% if entity.original_name %} + {{ entity.original_name }}
+ {% endif %} {% if entity.publisher %} -
{{ entity.publisher }}
+ {{ entity.publisher }} +
{% endif %} {% if entity.issnl %} - issn:{{ entity.issnl }} + issnl:{{ entity.issnl }}   + {% endif %} + {% if entity.wikidata_qid %} + wikidata:{{ entity.wikidata_qid }}   + {% endif %} + {% if entity.dblp_prefix %} + dblp:{{ entity.dblp_prefix }}   + {% endif %} + {% if entity.ia_sim_collection %} + [archive.org]   + {% elif entity.sim_pubid %} + [archive.org]   + {% endif %} + {# too much clutter + {% if entity.country_code %} + country:{{ entity.country_code }}   + {% endif %} + {% for lang in entity.languages %} + lang:{{ lang }}   + {% endfor %} + {% if entity.any_kbart %} + [KBART]   + {% endif %} +
+ #} + {% if entity.in_doaj %} + [DOAJ]   + {% endif %} + {% if entity.in_road %} + [ROAD]   + {% endif %} + {% if entity.is_oa and not (entity.in_doaj or entity.in_road) %} + [open-access]   {% endif %} - {% if entity.container_type %} -  {{ entity.container_type }} + {% if entity.container_type and entity.container_type != "journal" %} + [{{ entity.container_type }}]   {% endif %} {% if entity.publication_status and entity.publication_status != "active" %} -  {{ entity.publication_status }} + {{ entity.publication_status }}   {% endif %}
{% endmacro %} -- cgit v1.2.3 From 6976b6868cdf0628aa79d47aab4e889a9ccfc0dc Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 8 Feb 2022 19:02:34 -0800 Subject: container SERP: fix layout on mobile --- python/fatcat_web/templates/container_search.html | 7 +- python/fatcat_web/templates/entity_macros.html | 109 +++++++++++----------- 2 files changed, 62 insertions(+), 54 deletions(-) (limited to 'python/fatcat_web/templates/entity_macros.html') diff --git a/python/fatcat_web/templates/container_search.html b/python/fatcat_web/templates/container_search.html index f091fc3b..0031a0a8 100644 --- a/python/fatcat_web/templates/container_search.html +++ b/python/fatcat_web/templates/container_search.html @@ -33,7 +33,12 @@ {% if found %} {% if found.results %} - {{ search_macros.top_results(query, found) }} +
+
+ Content Status +
+ {{ search_macros.top_results(query, found) }} +

{% for entity in found.results %} diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index b801f3c5..6ad63a14 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -268,9 +268,63 @@ {% macro container_search_result_row(entity, show_stats=False) -%} -
+
+
+

+ {{ entity['name'] }} +

+ {% if entity.original_name %} + {{ entity.original_name }}
+ {% endif %} + {% if entity.publisher %} + {{ entity.publisher }} +
+ {% endif %} + {% if entity.issnl %} + issnl:{{ entity.issnl }}   + {% endif %} + {% if entity.wikidata_qid %} + wikidata:{{ entity.wikidata_qid }}   + {% endif %} + {% if entity.dblp_prefix %} + dblp:{{ entity.dblp_prefix }}   + {% endif %} + {% if entity.ia_sim_collection %} + [archive.org]   + {% elif entity.sim_pubid %} + [archive.org]   + {% endif %} + {# too much clutter + {% if entity.country_code %} + country:{{ entity.country_code }}   + {% endif %} + {% for lang in entity.languages %} + lang:{{ lang }}   + {% endfor %} + {% if entity.any_kbart %} + [KBART]   + {% endif %} +
+ #} + {% if entity.in_doaj %} + [DOAJ]   + {% endif %} + {% if entity.in_road %} + [ROAD]   + {% endif %} + {% if entity.is_oa and not (entity.in_doaj or entity.in_road) %} + [open-access]   + {% endif %} + {% if entity.container_type and entity.container_type != "journal" %} + [{{ entity.container_type }}]   + {% endif %} + {% if entity.publication_status and entity.publication_status != "active" %} + {{ entity.publication_status }}   + {% endif %} +
+ {% if show_stats %} -
+
{% if entity.releases_total %} ~{{ "{:,}".format(entity.releases_total) }} releases {% elif entity.releases_total == 0 %} @@ -281,57 +335,6 @@ {% endif %}
{% endif %} -

- {{ entity['name'] }} -

- {% if entity.original_name %} - {{ entity.original_name }}
- {% endif %} - {% if entity.publisher %} - {{ entity.publisher }} -
- {% endif %} - {% if entity.issnl %} - issnl:{{ entity.issnl }}   - {% endif %} - {% if entity.wikidata_qid %} - wikidata:{{ entity.wikidata_qid }}   - {% endif %} - {% if entity.dblp_prefix %} - dblp:{{ entity.dblp_prefix }}   - {% endif %} - {% if entity.ia_sim_collection %} - [archive.org]   - {% elif entity.sim_pubid %} - [archive.org]   - {% endif %} - {# too much clutter - {% if entity.country_code %} - country:{{ entity.country_code }}   - {% endif %} - {% for lang in entity.languages %} - lang:{{ lang }}   - {% endfor %} - {% if entity.any_kbart %} - [KBART]   - {% endif %} -
- #} - {% if entity.in_doaj %} - [DOAJ]   - {% endif %} - {% if entity.in_road %} - [ROAD]   - {% endif %} - {% if entity.is_oa and not (entity.in_doaj or entity.in_road) %} - [open-access]   - {% endif %} - {% if entity.container_type and entity.container_type != "journal" %} - [{{ entity.container_type }}]   - {% endif %} - {% if entity.publication_status and entity.publication_status != "active" %} - {{ entity.publication_status }}   - {% endif %}
{% endmacro %} -- cgit v1.2.3 From 9b9ede257753b1e3fd19bbf90a5895de5db18504 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 11 Feb 2022 11:14:07 -0800 Subject: containers: progress on browse interface --- python/fatcat_web/routes.py | 92 ++++++++++++++++++++-- python/fatcat_web/search.py | 7 +- .../templates/container_view_browse.html | 71 ++++++++++++----- python/fatcat_web/templates/entity_macros.html | 4 +- 4 files changed, 146 insertions(+), 28 deletions(-) (limited to 'python/fatcat_web/templates/entity_macros.html') diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 0afc189f..6e3e9b57 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -287,8 +287,6 @@ def generic_entity_view(entity_type: str, ident: str, view_template: str) -> Any entity._type_preservation = get_elastic_preservation_by_type( ReleaseQuery(container_id=ident), ) - if view_template == "container_view_browse.html": - entity._browse_volume_year = get_elastic_container_browse_year_volume(entity.ident) return render_template( view_template, entity_type=entity_type, entity=entity, editgroup_id=None @@ -350,9 +348,93 @@ def container_view_coverage(ident: str) -> AnyResponse: @app.route("/container//browse", methods=["GET"]) -def container_view_browser(ident: str) -> AnyResponse: - # note: there is a special hack to add entity._type_preservation for this endpoint - return generic_entity_view("container", ident, "container_view_browse.html") +def container_view_browse(ident: str) -> AnyResponse: + entity = generic_get_entity("container", ident) + + if entity.state == "redirect": + return redirect(f"/container/{entity.redirect}") + elif entity.state == "deleted": + return render_template("deleted_entity.html", entity_type="container", entity=entity) + + query_sort: Optional[List[str]] + if request.args.get('year') and 'volume' in request.args and 'issue' in request.args: + # year, volume, issue specified; browse-by-page + year = int(request.args.get('year')) + volume = request.args.get('volume', '') + issue = request.args.get('issue', '') + if volume: + volume = f'volume:"{volume}"' + else: + volume = "!volume:*" + if issue: + issue = f'issue:"{issue}"' + else: + issue = "!issue:*" + query_string = f'year:{year} {volume} {issue}' + query_sort = ["first_page", "release_date"] + elif request.args.get('year') and 'volume' in request.args: + # year, volume specified (no issue); browse-by-page + year = int(request.args.get('year')) + volume = request.args.get('volume', '') + if volume: + volume = f'volume:"{volume}"' + else: + volume = "!volume:*" + query_string = f'year:{year} {volume}' + query_sort = ["issue", "first_page", "release_date"] + elif request.args.get('year'): + # year specified, not anything else; browse-by-date + year = int(request.args.get('year')) + query_string = f"year:{year}" + query_sort = ["release_date"] + else: + entity._browse_volume_year = get_elastic_container_browse_year_volume(entity.ident) + return render_template( + "container_view_browse.html", entity_type="container", entity=entity, editgroup_id=None + ) + + print(query_string) + query = ReleaseQuery( + q=query_string, + limit=200, + offset=0, + container_id=ident, + fulltext_only=False, + recent=False, + exclude_stubs=True, + sort=query_sort, + ) + + try: + found = do_release_search(query) + except FatcatSearchError as fse: + return ( + render_template( + "container_view_search.html", + query=query, + es_error=fse, + entity_type="container", + entity=entity, + editgroup_id=None, + ), + fse.status_code, + ) + + # HACK: re-sort by first page *numerically* + if found.results and query_sort and 'first_page' in query_sort: + for doc in found.results: + if doc.get('first_page') and doc['first_page'].isdigit(): + doc['first_page'] = int(doc['first_page']) + found.results = sorted(found.results, key=lambda d: d.get('first_page') or 99999999) + + return render_template( + "container_view_browse.html", + query=query, + releases_found=found, + entity_type="container", + entity=entity, + editgroup_id=None, + ) @app.route("/container//metadata", methods=["GET"]) diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index b19b57e6..3f4a216c 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -29,6 +29,7 @@ class ReleaseQuery: container_id: Optional[str] = None recent: bool = False exclude_stubs: bool = False + sort: Optional[List[str]] = None @staticmethod def from_args(args: Dict[str, Any]) -> "ReleaseQuery": @@ -45,6 +46,7 @@ class ReleaseQuery: container_id=args.get("container_id"), recent=bool(args.get("recent")), exclude_stubs=bool(args.get("exclude_stubs")), + sort=None, ) @@ -182,6 +184,9 @@ def do_release_search(query: ReleaseQuery, deep_page_limit: int = 2000) -> Searc negative_boost=0.5, ) + if query.sort: + search = search.sort(*query.sort) + # Sanity checks limit = min((int(query.limit or 25), 100)) offset = max((int(query.offset or 0), 0)) @@ -299,7 +304,7 @@ def get_elastic_container_browse_year_volume(ident: str) -> Dict[int, Dict[str, for year in year_nums: year_dicts[year] = {} for row in buckets: - year_dicts[int(row["key"]["year"])][row["key"]["volume"] or "_unknown"] = int( + year_dicts[int(row["key"]["year"])][row["key"]["volume"] or "000_unknown"] = int( row["doc_count"] ) # return sorted(year_dicts.values(), key=lambda x: x["year"]) diff --git a/python/fatcat_web/templates/container_view_browse.html b/python/fatcat_web/templates/container_view_browse.html index b5691899..aa88b666 100644 --- a/python/fatcat_web/templates/container_view_browse.html +++ b/python/fatcat_web/templates/container_view_browse.html @@ -4,28 +4,59 @@ {% import "entity_macros.html" as entity_macros %} {% extends "entity_base.html" %} -{% block entity_main %} - -{% if entity._browse_volume_year %} -

Browse by Year and Volume

-
    -{% for year in entity._browse_volume_year.keys()|sort|reverse %} - {% for volume in entity._browse_volume_year[year].keys()|sort|reverse %} - {% if volume == '_unknown' %} -
  • {{ year }} ({{ entity._browse_volume_year[year][volume] }} releases) - {% else %} -
  • {{ year }} | Vol. {{ volume }} ({{ entity._browse_volume_year[year][volume] }} releases) - {% endif %} +{% macro browse_year_volume_table(entity, data) %} + + + {% for year in data.keys()|sort|reverse %} + {% for volume in data[year].keys()|sort|reverse %} + + {% if loop.first %} + + {% endif %} + {% if volume == '000_unknown' %} + + + + {% endfor %} {% endfor %} -{% endfor %} - -{% elif entity._browse_issues %} -{% for issue in entity._browse_issues.keys()|sort|reverse %} -

    {{ issue }}

    - {% for paper in entity._browse_issues[issue] %} - {{ paper.title }}
    + +
    + {{ year }} + {% if data[year]|length > 1 %}n/a{% endif %}{{ data[year][volume] }} releases + {% else %} + Vol. {{ volume }}{{ data[year][volume] }} releases + {% endif %} +
    +{% endmacro %} + +{% macro browse_releases(found) %} + {% for release_doc in found.results %} +
    +
    + {% if release_doc.pages %} + {{ release_doc.pages }} + {# + {% elif release_doc.release_date %} + {{ release_doc.release_date }} + #} + {% endif %} +
    +
    + {{ entity_macros.release_search_result_row(release_doc, margin_top=False) }} +
    +
    {% endfor %} -{% endfor %} +{% endmacro %} + +{% block entity_main %} + +{% if releases_found %} + {{ browse_releases(releases_found) }} +{% elif entity._browse_volume_year %} +
    +

    Contents by Year and Volume

    + {{ browse_year_volume_table(entity, entity._browse_volume_year) }} +
    {% endif %} {% endblock %} diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 6ad63a14..163af3e0 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -142,9 +142,9 @@ {%- endmacro %} -{% macro release_search_result_row(paper) -%} +{% macro release_search_result_row(paper, margin_top=True) -%}
    -

    +

    {% if paper.title %} {{ paper.title[:512] }} -- cgit v1.2.3 From 9d668d2632bd969cfd850650641a76f56a1dbfab Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 15 Feb 2022 18:15:14 -0800 Subject: web: small improvements to release SERP rows --- python/fatcat_web/templates/entity_macros.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'python/fatcat_web/templates/entity_macros.html') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 163af3e0..c510a8b9 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -224,7 +224,7 @@ {% endif %} {% if paper.withdrawn_status %} - {{ paper.withdrawn_status }} +   [{{ paper.withdrawn_status }}] {% endif %} {# ### IDENTIFIERS #} @@ -257,6 +257,9 @@ {% if paper.wikidata_qid %} wikidata:{{ paper.wikidata_qid }}   {% endif %} + {% if paper.version %} + version:{{ paper.version }}   + {% endif %} {# WIP: elastic release work grouping searches
    -- cgit v1.2.3 From ae97f4d8d4446d32b07efba587b57b741a16cfec Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 15 Feb 2022 20:51:01 -0800 Subject: containers: update preservation page Mostly adding copy and a KBART holdings page. --- python/fatcat_web/templates/container_view.html | 33 ----- .../templates/container_view_coverage.html | 134 ++++++++++++++------- python/fatcat_web/templates/entity_macros.html | 15 ++- 3 files changed, 101 insertions(+), 81 deletions(-) (limited to 'python/fatcat_web/templates/entity_macros.html') diff --git a/python/fatcat_web/templates/container_view.html b/python/fatcat_web/templates/container_view.html index db458589..0432a12d 100644 --- a/python/fatcat_web/templates/container_view.html +++ b/python/fatcat_web/templates/container_view.html @@ -158,39 +158,6 @@

    {% endif %} -{# -{%- if container.extra and container.extra.kbart %} -
    -Preservation Holdings
    - {% for k, v in container.extra.kbart.items() %} - {% if k == "lockss" %} - LOCKSS: - {% elif k == "clockss" %} - CLOCKSS: - {% elif k == "hathitrust" and container.issnl %} - HathiTrust: - {% elif k == "portico" and container.issnl %} - Portico: - {% elif k == "scholarsportal" and container.issnl %} - Scholars Portal: - {% else %} - {{ k }}: - {% endif %} - years - {% for span in v.year_spans %} - {% if span|length >= 2 -%} - {{ span[0] }}-{{ span[1] -}} - {% elif span|length == 1 -%} - {{ span[0] -}} - {% endif -%} - {{- ", " if not loop.last }} - {% endfor %} -
    - {% endfor %} -
    -{% endif %} -#} -
    Lookup Links
    diff --git a/python/fatcat_web/templates/container_view_coverage.html b/python/fatcat_web/templates/container_view_coverage.html index 3022c0d9..865723d5 100644 --- a/python/fatcat_web/templates/container_view_coverage.html +++ b/python/fatcat_web/templates/container_view_coverage.html @@ -6,35 +6,79 @@ {% block entity_main %} -
    -
    -
    -
    -
    {{ "{:,}".format(container._stats.total) }}
    -
    Known Releases
    -
    - {% if container._stats.total >= 1 %} - {{ entity_macros.preservation_bar(container._stats.preservation, extra_class="large") }} - {{ entity_macros.preservation_table(container._stats.preservation) }} - {% endif %} -

    - {% if container.extra and container.extra.kbart %} - There seem to be at least some "dark" preservation holdings in: - {% for k, v in container.extra.kbart.items() %} - {{ k }}{{ ", " if not loop.last }} - {% endfor %} - . - {% endif %} - {% if container.issnl %} - Our metadata may not be up to date, but you can verify preservation holdings in Keepers Registery (click "Archival Status"). - {% endif %} -

    +
    +
    +

    This page summarizes and visualizes the article-level (or output-level) preservation status for a single publication venue ({{ container.name }}). Fatcat is a preservation-oriented catalog, and metadata is aggregated from many sources. +

    However, metadata quality and consistency is difficult at scale and there may be preservation coverage not recorded here, or in some rare cases we may have content incorrectly matched and marked as preserved. Please contact us or submit corrections directly if you find any mistakes or inaccuracies.

    {% if container._stats.total >= 1 %} -

    -

    Preservation Coverage by Year

    +
    +

    Overall Preservation Coverage

    + {{ entity_macros.preservation_bar(container._stats.preservation, extra_class="large") }} +
    +
    + {{ entity_macros.preservation_table(container._stats.preservation) }} +
    +
    +{% endif %} + +
    +

    Known Holdings Elsewhere

    +

    This table is based on KBART reports from large, independent, long-term digital preservation projects. We use the start and stop years of fulltext coverage, then count individuals works as included or not on the basis of year alone (not considering volume or issue metadata). These are mostly "dark" archives, with no direct public access to holdings. +

    The Keeper's Registry project, currently run by issn.org, is a more authoritative source for aggregated KBART reports, and includes more archives. + {% if container.issnl %} + You can double check the Keeper's entry for this ISSN on portal.issn.org; click through to the "Archival Status" link to see holdings information. + {% endif %} + + {% if container.extra and container.extra.kbart %} + + + + + + + + {% for k, v in container.extra.kbart.items() %} + + + +
    ArchiveYear Span(s)
    + {% if k == "lockss" %} + LOCKSS + {% elif k == "clockss" %} + CLOCKSS + {% elif k == "hathitrust" and container.issnl %} + HathiTrust + {% elif k == "portico" and container.issnl %} + Portico + {% elif k == "scholarsportal" and container.issnl %} + Scholars Portal + {% else %} + {{ k }} + {% endif %} + + {% for span in v.year_spans %} + {% if span|length >= 2 -%} + {{ span[0] }} to {{ span[1] -}} + {% elif span|length == 1 -%} + {{ span[0] -}} + {% endif -%} +
    + {% endfor %} + {% if not v.year_spans %}-{% endif %} + {% endfor %} +
    + {% else %} +

    No holdings at any other locations recorded. + {% endif %} +

    + +{% if container._stats.total >= 1 %} +
    +

    Preservation Coverage by Year

    +
    @@ -45,7 +89,9 @@


    -

    Preservation Coverage by Volume

    +
    +

    Preservation Coverage by Volume Number

    +
    @@ -56,23 +102,25 @@


    -

    Preservation Coverage by Release Type

    - - - - - - {% for type_row in container._type_preservation %} - - -
    Release Type - Total Count - Coverage -
    {{ type_row.release_type }} - {{ "{:,}".format(type_row.total) }} - {{ entity_macros.preservation_bar(type_row) }} - {% endfor %} -
    +
    +

    Preservation Coverage by Release Type

    + + + + + + {% for type_row in container._type_preservation %} + + +
    Release Type + Total Count + Coverage +
    {{ type_row.release_type }} + {{ "{:,}".format(type_row.total) }} + {{ entity_macros.preservation_bar(type_row) }} + {% endfor %} +
    +
    {% endif %} {% endblock %} diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index c510a8b9..5f8f6e0a 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -388,31 +388,36 @@ yellow {% set frac_dark = stats.dark/stats.total %} {% set frac_none = stats.none/stats.total %} - +
    +
    - {{ "{:,}".format(stats.bright) }} + {{ "{:,}".format(stats.bright) }} {{ (frac_bright*100)|round(2,method='ceil') }}% preserved and publicly accessible (bright)
    - {{ "{:,}".format(stats.dark) }} + {{ "{:,}".format(stats.dark) }} {{ (frac_dark*100)|round(2,method='ceil') }}% preserved but not publicly accessible (dark) {% if stats.shadows_only %} {% set frac_shadows_only = stats.shadows_only/stats.total %}
    - {{ "{:,}".format(stats.shadows_only) }} + {{ "{:,}".format(stats.shadows_only) }} {{ (frac_shadows_only*100)|round(2,method='ceil') }}% only independently preserved in "shadow" libraries {% endif %}
    - {{ "{:,}".format(stats.none) }} + {{ "{:,}".format(stats.none) }} {{ (frac_none*100)|round(2,method='ceil') }}% no known independent preservation +
    + {{ "{:,}".format(stats.total) }} + + total
    -- cgit v1.2.3