From ed72027bbf36e933c8db069bd02b0163a84aef83 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Fri, 8 Nov 2019 23:00:29 +0100 Subject: Add basic pagination to search results The "deep paging problem" imposes some limit, which currently is a hardcoded default value, `deep_page_limit=2000` in `do_search`. Elasticsearch can be configured, too: > Note that from + size can not be more than the index.max_result_window index setting, which defaults to 10,000. -- https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-from-size --- python/fatcat_web/routes.py | 9 ++++++-- python/fatcat_web/search.py | 27 +++++++++++++++-------- python/fatcat_web/templates/container_search.html | 21 +++++++++++++++++- python/fatcat_web/templates/release_search.html | 24 ++++++++++++++++++-- 4 files changed, 67 insertions(+), 14 deletions(-) (limited to 'python') diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 79b594e3..a41f388d 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -673,9 +673,12 @@ def release_search(): if container_id and query: query += ' container_id:"{}"'.format(container_id) + offset = request.args.get('offset', '0') + offset = max(0, int(offset)) if offset.isnumeric() else 0 + if 'q' in request.args.keys(): # always do files for HTML - found = do_release_search(query, fulltext_only=fulltext_only) + found = do_release_search(query, fulltext_only=fulltext_only, offset=offset) return render_template('release_search.html', found=found, query=query, fulltext_only=fulltext_only) else: return render_template('release_search.html', query=query, fulltext_only=fulltext_only) @@ -684,10 +687,12 @@ def release_search(): def container_search(): query = request.args.get('q') + offset = request.args.get('offset', '0') + offset = max(0, int(offset)) if offset.isnumeric() else 0 if 'q' in request.args.keys(): # always do files for HTML - found = do_container_search(query) + found = do_container_search(query, offset=offset) return render_template('container_search.html', found=found, query=query) else: return render_template('container_search.html', query=query) diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 52f05f71..7c60a6dd 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -11,15 +11,20 @@ import requests from flask import abort, flash from fatcat_web import app +def do_search(index, request, limit=30, offset=0, deep_page_limit=2000): -def do_search(index, request, limit=30): - + # Sanity checks if limit > 100: - # Sanity check limit = 100 + if offset < 0: + offset = 0 + if offset > deep_page_limit: + # Avoid deep paging problem. + offset = deep_page_limit request["size"] = int(limit) - #print(request) + request["from"] = int(offset) + # print(request) resp = requests.get("%s/%s/_search" % (app.config['ELASTICSEARCH_BACKEND'], index), json=request) @@ -45,10 +50,12 @@ def do_search(index, request, limit=30): return {"count_returned": len(results), "count_found": content['hits']['total'], - "results": results } + "results": results, + "offset": offset, + "deep_page_limit": deep_page_limit} -def do_release_search(q, limit=30, fulltext_only=True): +def do_release_search(q, limit=30, fulltext_only=True, offset=0): #print("Search hit: " + q) if limit > 100: @@ -75,17 +82,18 @@ def do_release_search(q, limit=30, fulltext_only=True): }, } - resp = do_search(app.config['ELASTICSEARCH_RELEASE_INDEX'], search_request) + resp = do_search(app.config['ELASTICSEARCH_RELEASE_INDEX'], search_request, offset=offset) for h in resp['results']: # Ensure 'contrib_names' is a list, not a single string if type(h['contrib_names']) is not list: h['contrib_names'] = [h['contrib_names'], ] h['contrib_names'] = [name.encode('utf8', 'ignore').decode('utf8') for name in h['contrib_names']] resp["query"] = { "q": q } + resp["limit"] = limit return resp -def do_container_search(q, limit=30): +def do_container_search(q, limit=30, offset=0): # Convert raw ISSN-L to ISSN-L query if len(q.split()) == 1 and len(q) == 9 and q[0:4].isdigit() and q[4] == '-': @@ -103,8 +111,9 @@ def do_container_search(q, limit=30): }, } - resp = do_search(app.config['ELASTICSEARCH_CONTAINER_INDEX'], search_request, limit=limit) + resp = do_search(app.config['ELASTICSEARCH_CONTAINER_INDEX'], search_request, limit=limit, offset=offset) resp["query"] = { "q": q } + resp["limit"] = limit return resp def get_elastic_entity_stats(): diff --git a/python/fatcat_web/templates/container_search.html b/python/fatcat_web/templates/container_search.html index 7f6799dd..9e2aa10a 100644 --- a/python/fatcat_web/templates/container_search.html +++ b/python/fatcat_web/templates/container_search.html @@ -30,7 +30,26 @@ {% if found %} {% if found.results %} - Showing top {{ found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} + Showing + {% if found.offset == 0 %} + first + {% else %} + results {{ found.offset }} — + {% endif %} + {{ found.offset + found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} +
+ {% if found.offset > 0 %} + {% if found.offset - found.limit < 0 %} + Prev + {% else %} + Prev + {% endif %} + {% endif %} + + {% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} + Next + {% endif %} + {% for entity in found.results %}

diff --git a/python/fatcat_web/templates/release_search.html b/python/fatcat_web/templates/release_search.html index 7d6b0443..359038dc 100644 --- a/python/fatcat_web/templates/release_search.html +++ b/python/fatcat_web/templates/release_search.html @@ -36,8 +36,28 @@ {% if found %} {% if found.results %} - Showing top {{ found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} -{% for paper in found.results %} + Showing + {% if found.offset == 0 %} + first + {% else %} + results {{ found.offset }} — + {% endif %} + {{ found.offset + found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} + +
+ {% if found.offset > 0 %} + {% if found.offset - found.limit < 0 %} + Prev + {% else %} + Prev + {% endif %} + {% endif %} + + {% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} + Next + {% endif %} + + {% for paper in found.results %} {{ entity_macros.release_search_result_row(paper) }} {% endfor %} {% if found.results|length > 8 %} -- cgit v1.2.3 From 116a47f6057245159851ff204f9e4c1e643c3d8d Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Wed, 13 Nov 2019 16:58:09 +0100 Subject: move pagination into macros Two new macros: * top_results(found) * bottom_results(found) wip: move pagination into macro --- python/fatcat_web/templates/container_search.html | 27 ++++----------- python/fatcat_web/templates/entity_macros.html | 40 +++++++++++++++++++++++ python/fatcat_web/templates/release_search.html | 27 +++------------ 3 files changed, 51 insertions(+), 43 deletions(-) (limited to 'python') diff --git a/python/fatcat_web/templates/container_search.html b/python/fatcat_web/templates/container_search.html index 9e2aa10a..48c613f8 100644 --- a/python/fatcat_web/templates/container_search.html +++ b/python/fatcat_web/templates/container_search.html @@ -1,3 +1,4 @@ +{% import "entity_macros.html" as entity_macros %} {% extends "base.html" %} {% block title %} @@ -30,25 +31,7 @@ {% if found %} {% if found.results %} - Showing - {% if found.offset == 0 %} - first - {% else %} - results {{ found.offset }} — - {% endif %} - {{ found.offset + found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} -
- {% if found.offset > 0 %} - {% if found.offset - found.limit < 0 %} - Prev - {% else %} - Prev - {% endif %} - {% endif %} - - {% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} - Next - {% endif %} + {{ entity_macros.top_results(found) }} {% for entity in found.results %}
@@ -68,8 +51,10 @@
{% endfor %} {% if found.results|length > 8 %} -
- Showing top {{ found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} +
+
+ {{ entity_macros.bottom_results(found)}} +
{% endif %} {% else %} diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index fc199c21..5e980fec 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -234,3 +234,43 @@ yellow {% endif %} {%- endmacro %} +{% macro top_results(found) -%} + +Showing + {% if found.offset == 0 %} + first + {% else %} + results {{ found.offset }} — + {% endif %} + + {{ found.offset + found.count_returned }} + out of {{ found.count_found }} results + + +{%- endmacro %} + + +{% macro bottom_results(found) -%} + +{% if found.offset > 0 %} + {% if found.offset - found.limit < 0 %} + « Previous + {% else %} + « Previous + {% endif %} +{% else %} + « Previous +{% endif %} + +  Showing results {{ found.offset }} — {{ found.offset + +found.count_returned }} out of {{ found.count_found }} results   + +{% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} + Next » + {% else %} + Next » +{% endif %} + +

+ +{%- endmacro %} \ No newline at end of file diff --git a/python/fatcat_web/templates/release_search.html b/python/fatcat_web/templates/release_search.html index 359038dc..47f4330a 100644 --- a/python/fatcat_web/templates/release_search.html +++ b/python/fatcat_web/templates/release_search.html @@ -36,33 +36,16 @@ {% if found %} {% if found.results %} - Showing - {% if found.offset == 0 %} - first - {% else %} - results {{ found.offset }} — - {% endif %} - {{ found.offset + found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} - -
- {% if found.offset > 0 %} - {% if found.offset - found.limit < 0 %} - Prev - {% else %} - Prev - {% endif %} - {% endif %} - - {% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} - Next - {% endif %} + {{ entity_macros.top_results(found) }} {% for paper in found.results %} {{ entity_macros.release_search_result_row(paper) }} {% endfor %} {% if found.results|length > 8 %} -
- Showing top {{ found.count_returned }} out of {{ found.count_found }} results for: {{ found.query.q }} +
+
+ {{ entity_macros.bottom_results(found)}} +
{% endif %} {% else %} -- cgit v1.2.3 From 6956dc39c2df9a6db5b6b402b931cf5baf586232 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Thu, 14 Nov 2019 15:02:22 +0100 Subject: gray out inactive navigation links As per [this issue](https://github.com/Semantic-Org/Semantic-UI/issues/1885#issuecomment-77619519), text colors are not supported in semantic ui. To not move text too much, gray out inactive links. --- python/fatcat_web/templates/entity_macros.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 5e980fec..390327bc 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -259,7 +259,7 @@ yellow « Previous {% endif %} {% else %} - « Previous + « Previous {% endif %}   Showing results {{ found.offset }} — {{ found.offset + @@ -268,7 +268,7 @@ found.count_returned }} out of {{ found.count_found }} results   {% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} Next » {% else %} - Next » + Next » {% endif %} -- cgit v1.2.3 From cd32f2b7283d0d2735bb26cac9481556e10eab09 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Thu, 14 Nov 2019 16:14:14 +0100 Subject: adjust search test case for new wording > "Showing top " -> "Showing first " --- python/tests/web_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/tests/web_search.py b/python/tests/web_search.py index bdd858d0..4096a69c 100644 --- a/python/tests/web_search.py +++ b/python/tests/web_search.py @@ -16,7 +16,7 @@ def test_release_search(app): rv = app.get('/release/search?q=blood') assert rv.status_code == 200 - assert b"Showing top " in rv.data + assert b"Showing first " in rv.data @responses.activate def test_container_search(app): @@ -29,7 +29,7 @@ def test_container_search(app): rv = app.get('/container/search?q=blood') assert rv.status_code == 200 - assert b"Showing top " in rv.data + assert b"Showing first " in rv.data assert b"European Instructional Course Lectures" in rv.data assert b"British Editorial Society of Bone and Joint Surger" in rv.data -- cgit v1.2.3 From 0e4d65c773f586cc5cdafe8049e344a26688f710 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Fri, 15 Nov 2019 21:48:31 +0100 Subject: address test issue --- python/tests/web_search.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/tests/web_search.py b/python/tests/web_search.py index 4096a69c..19e2c29f 100644 --- a/python/tests/web_search.py +++ b/python/tests/web_search.py @@ -16,7 +16,8 @@ def test_release_search(app): rv = app.get('/release/search?q=blood') assert rv.status_code == 200 - assert b"Showing first " in rv.data + assert b"Showing" in rv.data + assert b"Quantum Studies of Acetylene Adsorption on Ice Surface" in rv.data @responses.activate def test_container_search(app): @@ -29,7 +30,7 @@ def test_container_search(app): rv = app.get('/container/search?q=blood') assert rv.status_code == 200 - assert b"Showing first " in rv.data + assert b"Showing" in rv.data assert b"European Instructional Course Lectures" in rv.data assert b"British Editorial Society of Bone and Joint Surger" in rv.data -- cgit v1.2.3