diff options
| -rw-r--r-- | python/fatcat_web/editing_routes.py | 20 | ||||
| -rw-r--r-- | python/fatcat_web/kafka.py | 2 | ||||
| -rw-r--r-- | python/fatcat_web/routes.py | 79 | ||||
| -rw-r--r-- | python/fatcat_web/search.py | 23 | ||||
| -rw-r--r-- | python/fatcat_web/templates/400.html | 18 | ||||
| -rw-r--r-- | python/fatcat_web/templates/api_error.html | 20 | ||||
| -rw-r--r-- | python/fatcat_web/templates/container_search.html | 8 | ||||
| -rw-r--r-- | python/fatcat_web/templates/entity_macros.html | 64 | ||||
| -rw-r--r-- | python/fatcat_web/templates/home.html | 6 | ||||
| -rw-r--r-- | python/fatcat_web/templates/release_search.html | 7 | ||||
| -rw-r--r-- | python/fatcat_web/templates/search_macros.html | 68 |
11 files changed, 208 insertions, 107 deletions
diff --git a/python/fatcat_web/editing_routes.py b/python/fatcat_web/editing_routes.py index 44000b1a..ffce35f3 100644 --- a/python/fatcat_web/editing_routes.py +++ b/python/fatcat_web/editing_routes.py @@ -25,7 +25,7 @@ def form_editgroup_get_or_create(api, edit_form): edit_form.editgroup_id.errors.append("Editgroup does not exist") return None app.log.warning(ae) - abort(ae.status) + raise ae if eg.changelog_index: edit_form.editgroup_id.errors.append("Editgroup has already been accepted") return None @@ -36,7 +36,7 @@ def form_editgroup_get_or_create(api, edit_form): Editgroup(description=edit_form.editgroup_description.data or None)) except ApiException as ae: app.log.warning(ae) - abort(ae.status) + raise ae # set this session editgroup_id flash('Started new editgroup <a href="/editgroup/{}">{}</a>'.format( eg.editgroup_id, @@ -79,8 +79,7 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template # check that editgroup is edit-able if editgroup.changelog_index != None: - flash("Editgroup already merged") - abort(400) + abort(400, "Editgroup already merged") # fetch entity (if set) or 404 existing = None @@ -123,7 +122,7 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template raise NotImplementedError except ApiException as ae: app.log.warning(ae) - abort(ae.status) + raise ae return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) else: # it's an update # all the tricky logic is in the update method @@ -151,7 +150,7 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template if ae.status == 404: pass else: - abort(ae.status) + raise ae try: if entity_type == 'container': edit = user_api.update_container(editgroup.editgroup_id, existing.ident, existing) @@ -163,7 +162,7 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template raise NotImplementedError except ApiException as ae: app.log.warning(ae) - abort(ae.status) + raise ae return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) else: status = 400 @@ -202,12 +201,11 @@ def generic_edit_delete(editgroup_id, entity_type, edit_id): try: editgroup = api.get_editgroup(editgroup_id) except ApiException as ae: - abort(ae.status) + raise ae # check that editgroup is edit-able if editgroup.changelog_index != None: - flash("Editgroup already merged") - abort(400) + abort(400, "Editgroup already merged") # API on behalf of user user_api = auth_api(session['api_token']) @@ -223,7 +221,7 @@ def generic_edit_delete(editgroup_id, entity_type, edit_id): else: raise NotImplementedError except ApiException as ae: - abort(ae.status) + raise ae return redirect("/editgroup/{}".format(editgroup_id)) diff --git a/python/fatcat_web/kafka.py b/python/fatcat_web/kafka.py index 44aac5fb..1d7288af 100644 --- a/python/fatcat_web/kafka.py +++ b/python/fatcat_web/kafka.py @@ -31,4 +31,4 @@ def kafka_pixy_produce(topic, msg, key=None, sync=True, timeout=25): timeout=timeout, ) resp.raise_for_status() - print(resp.json()) + #print(resp.json()) diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index 4a66b3c2..6f3ec21b 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -14,7 +14,7 @@ from fatcat_tools.normal import * from fatcat_web import app, api, auth_api, priv_api, mwoauth, Config from fatcat_web.auth import handle_token_login, handle_logout, load_user, handle_ia_xauth, handle_wmoauth from fatcat_web.cors import crossdomain -from fatcat_web.search import ReleaseQuery, GenericQuery, do_release_search, do_container_search, get_elastic_entity_stats, get_elastic_container_stats, get_elastic_container_histogram +from fatcat_web.search import ReleaseQuery, GenericQuery, do_release_search, do_container_search, get_elastic_entity_stats, get_elastic_container_stats, get_elastic_container_histogram, FatcatSearchError from fatcat_web.entity_helpers import * from fatcat_web.graphics import * from fatcat_web.kafka import * @@ -138,7 +138,7 @@ def generic_lookup_view(entity_type, lookup_template, extid_types, lookup_lambda ae.status) else: app.log.info(ae) - abort(ae.status) + raise ae return redirect('/{}/{}'.format(entity_type, resp.ident)) @app.route('/container/lookup', methods=['GET']) @@ -503,8 +503,7 @@ def editgroup_create_annotation(ident): try: eg = user_api.get_editgroup(str(ident)) if eg.changelog_index: - flash("Editgroup already accepted") - abort(400) + abort(400, "Editgroup already accepted") ega = EditgroupAnnotation( comment_markdown=comment_markdown, extra=None, @@ -512,7 +511,7 @@ def editgroup_create_annotation(ident): user_api.create_editgroup_annotation(eg.editgroup_id, ega) except ApiException as ae: app.log.info(ae) - abort(ae.status) + raise ae return redirect('/editgroup/{}'.format(ident)) @app.route('/editgroup/<ident>/accept', methods=['POST']) @@ -525,8 +524,7 @@ def editgroup_accept(ident): try: eg = user_api.get_editgroup(str(ident)) if eg.changelog_index: - flash("Editgroup already accepted") - abort(400) + abort(400, "Editgroup already accepted") user_api.accept_editgroup(str(ident)) except ApiException as ae: app.log.info(ae) @@ -543,8 +541,7 @@ def editgroup_unsubmit(ident): try: eg = user_api.get_editgroup(str(ident)) if eg.changelog_index: - flash("Editgroup already accepted") - abort(400) + abort(400, "Editgroup already accepted") user_api.update_editgroup(eg.editgroup_id, eg, submit=False) except ApiException as ae: app.log.info(ae) @@ -557,16 +554,13 @@ def editgroup_submit(ident): if not app.testing: app.csrf.protect() # on behalf of user... - print("submitting...") user_api = auth_api(session['api_token']) try: eg = user_api.get_editgroup(str(ident)) if eg.changelog_index: - flash("Editgroup already accepted") - abort(400) + abort(400, "Editgroup already accepted") user_api.update_editgroup(eg.editgroup_id, eg, submit=True) except ApiException as ae: - print(ae) app.log.info(ae) abort(ae.status) return redirect('/editgroup/{}'.format(ident)) @@ -655,7 +649,6 @@ def release_save(ident): json.dumps(msg, sort_keys=True), ) except Exception as e: - print(e, file=sys.stderr) return render_template('release_save.html', entity=release, form=form, spn_status='kafka-error'), 500 return render_template('release_save.html', entity=release, form=form, spn_status='success'), 200 elif form.errors: @@ -710,7 +703,10 @@ def release_search(): return render_template('release_search.html', query=ReleaseQuery(), found=None) query = ReleaseQuery.from_args(request.args) - found = do_release_search(query) + try: + found = do_release_search(query) + except FatcatSearchError as fse: + return render_template('release_search.html', query=query, es_error=fse), fse.status_code return render_template('release_search.html', query=query, found=found) @app.route('/container/search', methods=['GET', 'POST']) @@ -720,7 +716,10 @@ def container_search(): return render_template('container_search.html', query=GenericQuery(), found=None) query = GenericQuery.from_args(request.args) - found = do_container_search(query) + try: + found = do_container_search(query) + except FatcatSearchError as fse: + return render_template('container_search.html', query=query, es_error=fse), fse.status_code return render_template('container_search.html', query=query, found=found) def get_changelog_stats(): @@ -759,12 +758,11 @@ def stats_json(): @crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) def container_issnl_stats(issnl): if not (len(issnl) == 9 and issnl[4] == '-'): - flash("Not a valid ISSN-L: {}".format(issnl)) - abort(400) + abort(400, "Not a valid ISSN-L: {}".format(issnl)) try: container = api.lookup_container(issnl=issnl) except ApiException as ae: - abort(ae.status) + raise ae try: stats = get_elastic_container_stats(container.ident, issnl=container.issnl) except Exception as ae: @@ -820,7 +818,7 @@ def release_bibtex(ident): try: entity = api.get_release(ident) except ApiException as ae: - abort(ae.status) + raise ae csl = release_to_csl(entity) bibtex = citeproc_csl(csl, 'bibtex') return Response(bibtex, mimetype="text/plain") @@ -837,7 +835,7 @@ def release_citeproc(ident): try: entity = api.get_release(ident) except ApiException as ae: - abort(ae.status) + raise ae csl = release_to_csl(entity) cite = citeproc_csl(csl, style, is_html) if is_html: @@ -895,7 +893,7 @@ def change_username(): editor = user_api.update_editor(editor.editor_id, editor) except ApiException as ae: app.log.info(ae) - abort(ae.status) + raise ae # update our session session['editor'] = editor.to_dict() load_user(editor.editor_id) @@ -914,8 +912,7 @@ def create_auth_token(): duration_seconds = int(duration_seconds) assert duration_seconds >= 1 except (ValueError, AssertionError): - flash("duration_seconds must be a positive non-zero integer") - abort(400) + abort(400, "duration_seconds must be a positive non-zero integer") # check user's auth. api_token and editor_id are signed together in session # cookie, so if api_token is valid editor_id is assumed to match. If that @@ -932,7 +929,7 @@ def create_auth_token(): duration_seconds=duration_seconds) except ApiException as ae: app.log.info(ae) - abort(ae.status) + raise ae return render_template('auth_token.html', auth_token=resp.token) @app.route('/auth/logout') @@ -983,7 +980,7 @@ def page_method_not_allowed(e): @app.errorhandler(400) def page_bad_request(e): - return render_template('400.html'), 400 + return render_template('400.html', err=e), 400 @app.errorhandler(409) def page_edit_conflict(e): @@ -999,6 +996,36 @@ def page_server_error(e): def page_server_down(e): return render_template('503.html'), 503 +@app.errorhandler(ApiException) +def page_fatcat_api_error(ae): + """ + Generic error handler for fatcat API problems. With this error handler, + don't need to explicitly catch API exceptions: they should get caught and + routed correctly here. + """ + if ae.status == 404: + return page_not_found(ae) + elif ae.status in [401, 403]: + return page_not_authorized(ae) + elif ae.status in [405]: + return page_method_not_allowed(ae) + elif ae.status in [409]: + return page_edit_conflict(ae) + try: + json_body = json.loads(ae.body) + ae.error_name = json_body.get('error') + ae.message = json_body.get('message') + except ValueError: + pass + return render_template('api_error.html', api_error=ae), ae.status + +@app.errorhandler(ApiValueError) +def page_fatcat_api_value_error(ae): + ae.status = 400 + ae.error_name = "ValueError" + ae.message = str(ae) + return render_template('api_error.html', api_error=ae), 400 + @app.errorhandler(CSRFError) def page_csrf_error(e): return render_template('csrf_error.html', reason=e.description), 400 diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 55caa9c5..3fd7f9dc 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -15,6 +15,15 @@ import elasticsearch_dsl.response from fatcat_web import app +class FatcatSearchError(Exception): + + def __init__(self, status_code: int, name: str, description: str = None): + if status_code == "N/A": + status_code = 503 + self.status_code = status_code + self.name = name + self.description = description + @dataclass class ReleaseQuery: q: Optional[str] = None @@ -109,14 +118,19 @@ def wrap_es_execution(search: Search) -> Any: except elasticsearch.exceptions.RequestError as e: # this is a "user" error print("elasticsearch 400: " + str(e.info), file=sys.stderr) + description = None if e.info.get("error", {}).get("root_cause", {}): - raise ValueError(str(e.info["error"]["root_cause"][0].get("reason"))) - else: - raise ValueError(str(e.info)) + description = str(e.info["error"]["root_cause"][0].get("reason")) + raise FatcatSearchError(e.status_code, str(e.error), description) + except elasticsearch.exceptions.ConnectionError as e: + raise FatcatSearchError(e.status_code, "ConnectionError: search engine not available") except elasticsearch.exceptions.TransportError as e: # all other errors print("elasticsearch non-200 status code: {}".format(e.info), file=sys.stderr) - raise IOError(str(e.info)) + description = None + if e.info.get("error", {}).get("root_cause", {}): + description = str(e.info["error"]["root_cause"][0].get("reason")) + raise FatcatSearchError(e.status_code, str(e.error), description) return resp def do_container_search( @@ -192,6 +206,7 @@ def do_release_search( Q("bool", must_not=Q("exists", field="release_year")), Q("bool", must_not=Q("exists", field="release_type")), Q("bool", must_not=Q("exists", field="release_stage")), + Q("bool", must_not=Q("exists", field="container_id")), ], ) diff --git a/python/fatcat_web/templates/400.html b/python/fatcat_web/templates/400.html index f2659ca2..21f98aad 100644 --- a/python/fatcat_web/templates/400.html +++ b/python/fatcat_web/templates/400.html @@ -4,10 +4,20 @@ <center> <div style="font-size: 8em;">400</div> <div style="font-size: 3em;">Bad Request</div> - -<p>Wasn't able to handle the request, either due to incorrect or unexpected -input. Usually more context should be available; if you hit this page it means -you've discovered a new corner case! </center> +<div class="ui icon error message"> + <i class="ban icon"></i> + <div class="content"> + <div class="header"></div> + {% if err and err.description %} + <p>{{ err.description }} + {% else %} + <p>Wasn't able to handle the request, either due to incorrect or unexpected + input. Usually more context should be available; if you hit this page it means + you've discovered a new corner case! + {% endif %} + </div> +</div> + {% endblock %} diff --git a/python/fatcat_web/templates/api_error.html b/python/fatcat_web/templates/api_error.html new file mode 100644 index 00000000..1a44f610 --- /dev/null +++ b/python/fatcat_web/templates/api_error.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{% block body %} + +<center> +<div style="font-size: 8em;">{{ api_error.status or "" }}</div> +<div style="font-size: 3em;">{{ api_error.reason or "" }}</div> +</center> + +<div class="ui icon error message"> + <i class="ban icon"></i> + <div class="content"> + <div class="header"> + API Error: {{ api_error.error_name or "" }} + </div> + <p>{{ api_error.message or "" }} + </div> +</div> + + +{% endblock %} diff --git a/python/fatcat_web/templates/container_search.html b/python/fatcat_web/templates/container_search.html index 2566f542..bd92dc2b 100644 --- a/python/fatcat_web/templates/container_search.html +++ b/python/fatcat_web/templates/container_search.html @@ -1,4 +1,4 @@ -{% import "entity_macros.html" as entity_macros %} +{% import "search_macros.html" as search_macros %} {% extends "base.html" %} {% block title %} @@ -32,7 +32,7 @@ {% if found %} {% if found.results %} - {{ entity_macros.top_results(query, found) }} + {{ search_macros.top_results(query, found) }} {% for entity in found.results %} <div> @@ -55,7 +55,7 @@ {% if found.results|length > 8 %} <div class="ui divider"></div> <div style="text-align: center"> - {{ entity_macros.bottom_results(query, found, endpoint='container_search') }} + {{ search_macros.bottom_results(query, found, endpoint='container_search') }} </div> {% endif %} @@ -80,6 +80,8 @@ {% endif %} +{% elif es_error %} + {{ search_macros.es_error_msg(es_error) }} {% endif %} </div> diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html index 0e7f135a..8e4c4f6a 100644 --- a/python/fatcat_web/templates/entity_macros.html +++ b/python/fatcat_web/templates/entity_macros.html @@ -162,12 +162,10 @@ {# release type suffix #} {% if paper.release_type in ("article-journal", "paper-conference") %} {# pass #} - {% elif paper.release_type in ("book", "chapter", "dataset") %} - <b style="text-transform: uppercase;">[{{ paper.release_type }}]</b> {% elif not paper.release_type %} - <b style="text-transform: uppercase; color: black;">[unknown-media]</b> + <b style="text-transform: uppercase; color: black;">[unknown]</b> {% else %} - <b style="text-transform: uppercase;">[{{ paper.release_type }}]</b> + <b style="text-transform: uppercase; color: black;">[{{ paper.release_type }}]</b> {% endif %} {# show original_title #} @@ -212,17 +210,17 @@ {% endif %} {% if paper.container_is_oa %}<i class="icon unlock orange small"></i>{% endif %} {% endif %} - {% if paper.withdrawn_status %} - <b style="color: red;"><code>[{{ paper.withdrawn_status }}]</code></b> - {% endif %} - {% if paper.release_stage == "accepted" %} - <b style="color: darkmagenta;"><code>[{{ paper.release_stage }} manuscript]</code></b> - {% elif paper.release_stage == "submitted" %} - <b style="color: magenta;"><code>[pre-print]</code></b> + + {% if paper.release_stage == "submitted" %} + <b style="color: brown; text-transform: uppercase;">pre-print</b> {% elif paper.release_stage and paper.release_stage != "published" %} - <b style="color: magenta;"><code>[{{ paper.release_stage }}]</code></b> + <b style="color: brown; text-transform: uppercase;">{{ paper.release_stage }} version</b> {% elif not paper.release_stage %} - <b style="color: red;"><code>[unpublished?]</code></b> + <b style="color: brown; text-transform: uppercase;">unpublished</b> + {% endif %} + + {% if paper.withdrawn_status %} + <b style="color: red; text-transform: uppercase;">{{ paper.withdrawn_status }}</b> {% endif %} {# ### IDENTIFIERS #} @@ -262,43 +260,3 @@ yellow {% endif %} {%- endmacro %} -{% macro top_results(query, found) -%} - -<i>Showing - {% if found.offset == 0 %} - first - {% else %} - results {{ found.offset }} — - {% endif %} - - {{ found.offset + found.count_returned }} - out of {{ found.count_found }} results -</i> - -{%- endmacro %} - - -{% macro bottom_results(query, found, endpoint='release_search') -%} - -{% if found.offset > 0 %} - {% if found.offset - found.limit < 0 %} - <a href="{{ url_for(endpoint, q=query.q, offset=0) }}">« Previous</a> - {% else %} - <a href="{{ url_for(endpoint, q=query.q, offset=found.offset - found.limit) }}">« Previous</a> - {% endif %} -{% else %} - <span style="color:gray">« Previous</span> -{% endif %} - - <i>Showing results {{ found.offset }} — {{ found.offset + -found.count_returned }} out of {{ found.count_found }} results</i> - -{% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} - <a href="{{ url_for(endpoint, q=query.q, offset=found.offset + found.limit) }}">Next »</a> - {% else %} - <span style="color:gray">Next »</span> -{% endif %} - -</div> - -{%- endmacro %} diff --git a/python/fatcat_web/templates/home.html b/python/fatcat_web/templates/home.html index 698230d3..4557e212 100644 --- a/python/fatcat_web/templates/home.html +++ b/python/fatcat_web/templates/home.html @@ -35,17 +35,17 @@ <div class="row"> <div class="four wide mobile three wide center aligned column"> <a href="/stats" style="color: black;"> - <h4>106,283,000<br>Papers</h4> + <h4>110,814,532<br>Papers</h4> </a> </div> <div class="four wide mobile three wide center aligned column"> <a href="/stats" style="color: black;"> - <h4>23,036,825<br>Fulltext</h4> + <h4>26,173,743<br>Fulltext</h4> </a> </div> <div class="four wide mobile three wide center aligned column"> <a href="/stats" style="color: black;"> - <h4>148,757<br>Journals</h4> + <h4>151,707<br>Journals</h4> </a> </div> </div> diff --git a/python/fatcat_web/templates/release_search.html b/python/fatcat_web/templates/release_search.html index 58aa35d6..7fb475e3 100644 --- a/python/fatcat_web/templates/release_search.html +++ b/python/fatcat_web/templates/release_search.html @@ -1,4 +1,5 @@ {% import "entity_macros.html" as entity_macros %} +{% import "search_macros.html" as search_macros %} {% extends "base.html" %} {% block title %} @@ -37,7 +38,7 @@ {% if found %} {% if found.results %} - {{ entity_macros.top_results(query, found) }} + {{ search_macros.top_results(query, found) }} {% for paper in found.results %} {{ entity_macros.release_search_result_row(paper) }} @@ -46,7 +47,7 @@ {% if found.results|length > 8 %} <div class="ui divider"></div> <div style="text-align: center"> - {{ entity_macros.bottom_results(query, found, endpoint='release_search') }} + {{ search_macros.bottom_results(query, found, endpoint='release_search') }} </div> {% endif %} @@ -73,6 +74,8 @@ {% endif %} +{% elif es_error %} + {{ search_macros.es_error_msg(es_error) }} {% endif %} </div> diff --git a/python/fatcat_web/templates/search_macros.html b/python/fatcat_web/templates/search_macros.html new file mode 100644 index 00000000..a207bfbc --- /dev/null +++ b/python/fatcat_web/templates/search_macros.html @@ -0,0 +1,68 @@ + +{% macro top_results(query, found) -%} + +<i>Showing + {% if found.offset == 0 %} + first + {% else %} + results {{ found.offset }} — + {% endif %} + + {{ found.offset + found.count_returned }} + out of {{ found.count_found }} results +</i> + +{%- endmacro %} + + +{% macro bottom_results(query, found, endpoint='release_search') -%} + +{% if found.offset > 0 %} + {% if found.offset - found.limit < 0 %} + <a href="{{ url_for(endpoint, q=query.q, offset=0) }}">« Previous</a> + {% else %} + <a href="{{ url_for(endpoint, q=query.q, offset=found.offset - found.limit) }}">« Previous</a> + {% endif %} +{% else %} + <span style="color:gray">« Previous</span> +{% endif %} + + <i>Showing results {{ found.offset }} — {{ found.offset + +found.count_returned }} out of {{ found.count_found }} results</i> + +{% if found.offset + found.limit < found.count_found and found.offset + found.limit < found.deep_page_limit %} + <a href="{{ url_for(endpoint, q=query.q, offset=found.offset + found.limit) }}">Next »</a> + {% else %} + <span style="color:gray">Next »</span> +{% endif %} + +</div> + +{%- endmacro %} + + +{% macro es_error_msg(es_error) %} + <div class="ui icon error message"> + <i class="ban icon"></i> + <div class="content"> + <div class="header"> + {% if es_error.status_code == 400 %} + Query Error + {% else %} + Search Index Error + {% if es_error.status_code %}({{ es_error.status_code }}){% endif %} + {% endif %} + </div> + {% if es_error.description %} + <p>Computer said: <code>{{ es_error.description }}</code> + {% elif es_error.name %} + <p><b>{{ es_error.name }}</b> + {% endif %} + {% if es_error.status_code == 400 %} + <p>Query parsing is currently very naive. Sometimes you can fix this + problem by adding quotes around terms or entire phrases. + {% endif %} + </div> + </div> +{% endmacro %} + |