diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2021-11-02 18:14:43 -0700 | 
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2021-11-02 18:14:43 -0700 | 
| commit | 9dc891b8098542bb089c8c47098b60a8beb76a53 (patch) | |
| tree | 2c9b1c569b6b9bb4041ef51076d024b6980089c5 /python | |
| parent | 6464631dbe5c4afeb76f2f3c9d63b89f917c9a3b (diff) | |
| download | fatcat-9dc891b8098542bb089c8c47098b60a8beb76a53.tar.gz fatcat-9dc891b8098542bb089c8c47098b60a8beb76a53.zip | |
fmt (black): fatcat_web/
Diffstat (limited to 'python')
| -rw-r--r-- | python/fatcat_web/__init__.py | 19 | ||||
| -rw-r--r-- | python/fatcat_web/auth.py | 130 | ||||
| -rw-r--r-- | python/fatcat_web/cors.py | 33 | ||||
| -rw-r--r-- | python/fatcat_web/editing_routes.py | 592 | ||||
| -rw-r--r-- | python/fatcat_web/entity_helpers.py | 148 | ||||
| -rw-r--r-- | python/fatcat_web/forms.py | 455 | ||||
| -rw-r--r-- | python/fatcat_web/graphics.py | 135 | ||||
| -rw-r--r-- | python/fatcat_web/hacks.py | 12 | ||||
| -rw-r--r-- | python/fatcat_web/kafka.py | 7 | ||||
| -rw-r--r-- | python/fatcat_web/ref_routes.py | 175 | ||||
| -rw-r--r-- | python/fatcat_web/routes.py | 1003 | ||||
| -rw-r--r-- | python/fatcat_web/search.py | 518 | ||||
| -rw-r--r-- | python/fatcat_web/web_config.py | 62 | 
13 files changed, 1992 insertions, 1297 deletions
| diff --git a/python/fatcat_web/__init__.py b/python/fatcat_web/__init__.py index 336b4133..c0d0e0c8 100644 --- a/python/fatcat_web/__init__.py +++ b/python/fatcat_web/__init__.py @@ -1,4 +1,3 @@ -  import sys  import elasticsearch @@ -18,7 +17,7 @@ from raven.contrib.flask import Sentry  from fatcat_web.web_config import Config  toolbar = DebugToolbarExtension() -app = Flask(__name__, static_url_path='/static') +app = Flask(__name__, static_url_path="/static")  app.config.from_object(Config)  toolbar = DebugToolbarExtension(app)  FlaskUUID(app) @@ -26,7 +25,8 @@ app.csrf = CSRFProtect(app)  app.log = create_logger(app)  # This is the Markdown processor; setting default here -Misaka(app, +Misaka( +    app,      autolink=True,      no_intra_emphasis=True,      strikethrough=True, @@ -49,6 +49,7 @@ api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(conf))  app.jinja_env.trim_blocks = True  app.jinja_env.lstrip_blocks = True +  def auth_api(token):      conf = fatcat_openapi_client.Configuration()      conf.api_key["Authorization"] = token @@ -56,6 +57,7 @@ def auth_api(token):      conf.host = Config.FATCAT_API_HOST      return fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(conf)) +  if Config.FATCAT_API_AUTH_TOKEN:      print("Found and using privileged token (eg, for account signup)", file=sys.stderr)      priv_api = auth_api(Config.FATCAT_API_AUTH_TOKEN) @@ -69,9 +71,10 @@ else:  mwoauth = MWOAuth(      consumer_key=Config.WIKIPEDIA_CLIENT_ID or "dummy",      consumer_secret=Config.WIKIPEDIA_CLIENT_SECRET or "dummy", -    default_return_to='wp_oauth_finish_login') +    default_return_to="wp_oauth_finish_login", +)  mwoauth.handshaker.user_agent = "fatcat.wiki;python_web_interface" -app.register_blueprint(mwoauth.bp, url_prefix='/auth/wikipedia') +app.register_blueprint(mwoauth.bp, url_prefix="/auth/wikipedia")  app.es_client = elasticsearch.Elasticsearch(Config.ELASTICSEARCH_BACKEND) @@ -80,12 +83,12 @@ from fatcat_web import auth, cors, editing_routes, forms, ref_routes, routes  # TODO: blocking on ORCID support in loginpass  if Config.ORCID_CLIENT_ID:      orcid_bp = create_flask_blueprint(ORCiD, oauth, auth.handle_oauth) -    app.register_blueprint(orcid_bp, url_prefix='/auth/orcid') +    app.register_blueprint(orcid_bp, url_prefix="/auth/orcid")  if Config.GITLAB_CLIENT_ID:      gitlab_bp = create_flask_blueprint(Gitlab, oauth, auth.handle_oauth) -    app.register_blueprint(gitlab_bp, url_prefix='/auth/gitlab') +    app.register_blueprint(gitlab_bp, url_prefix="/auth/gitlab")  if Config.GITHUB_CLIENT_ID:      github_bp = create_flask_blueprint(GitHub, oauth, auth.handle_oauth) -    app.register_blueprint(github_bp, url_prefix='/auth/github') +    app.register_blueprint(github_bp, url_prefix="/auth/github") diff --git a/python/fatcat_web/auth.py b/python/fatcat_web/auth.py index 137bc2bb..4fe85770 100644 --- a/python/fatcat_web/auth.py +++ b/python/fatcat_web/auth.py @@ -1,4 +1,3 @@ -  from collections import namedtuple  import fatcat_openapi_client @@ -12,11 +11,12 @@ from fatcat_web import Config, api, app, login_manager, priv_api  def handle_logout():      logout_user() -    for k in ('editor', 'api_token'): +    for k in ("editor", "api_token"):          if k in session:              session.pop(k)      session.clear() +  def handle_token_login(token):      try:          m = pymacaroons.Macaroon.deserialize(token) @@ -33,22 +33,23 @@ def handle_token_login(token):      for caveat in m.first_party_caveats():          caveat = caveat.caveat_id          if caveat.startswith(b"editor_id = "): -            editor_id = caveat[12:].decode('utf-8') +            editor_id = caveat[12:].decode("utf-8")      if not editor_id:          app.log.warning("auth fail: editor_id missing in macaroon")          abort(400)      # fetch editor info      editor = api.get_editor(editor_id)      session.permanent = True  # pylint: disable=assigning-non-slot -    session['api_token'] = token -    session['editor'] = editor.to_dict() +    session["api_token"] = token +    session["editor"] = editor.to_dict()      login_user(load_user(editor.editor_id))      rp = "/auth/account" -    if session.get('next'): -        rp = session['next'] -        session.pop('next') +    if session.get("next"): +        rp = session["next"] +        session.pop("next")      return redirect(rp) +  # This will need to login/signup via fatcatd API, then set token in session  def handle_oauth(remote, token, user_info):      if user_info: @@ -57,22 +58,24 @@ def handle_oauth(remote, token, user_info):          # SUB is the stable internal identifier for the user (not usually the username itself)          # TODO: should have the real sub here          # TODO: would be nicer to pass preferred_username for account creation -        iss = remote.OAUTH_CONFIG['api_base_url'] +        iss = remote.OAUTH_CONFIG["api_base_url"]          # we reuse 'preferred_username' for account name auto-creation (but          # don't store it otherwise in the backend, at least currently). But i'm          # not sure all loginpass backends will set it -        if user_info.get('preferred_username'): -            preferred_username = user_info['preferred_username'] -        elif 'orcid.org' in iss: +        if user_info.get("preferred_username"): +            preferred_username = user_info["preferred_username"] +        elif "orcid.org" in iss:              # as a special case, prefix ORCiD identifier so it can be used as a              # username. If we instead used the human name, we could have              # collisions. Not a great user experience either way. -            preferred_username = 'i' + user_info['sub'].replace('-', '') +            preferred_username = "i" + user_info["sub"].replace("-", "")          else: -            preferred_username = user_info['sub'] +            preferred_username = user_info["sub"] -        params = fatcat_openapi_client.AuthOidc(remote.name, user_info['sub'], iss, preferred_username) +        params = fatcat_openapi_client.AuthOidc( +            remote.name, user_info["sub"], iss, preferred_username +        )          # this call requires admin privs          (resp, http_status, http_headers) = priv_api.auth_oidc_with_http_info(params)          editor = resp.editor @@ -80,90 +83,103 @@ def handle_oauth(remote, token, user_info):          # write token and username to session          session.permanent = True  # pylint: disable=assigning-non-slot -        session['api_token'] = api_token -        session['editor'] = editor.to_dict() +        session["api_token"] = api_token +        session["editor"] = editor.to_dict()          # call login_user(load_user(editor_id))          login_user(load_user(editor.editor_id))          rp = "/auth/account" -        if session.get('next'): -            rp = session['next'] -            session.pop('next') +        if session.get("next"): +            rp = session["next"] +            session.pop("next")          return redirect(rp)      # XXX: what should this actually be?      raise Exception("didn't receive OAuth user_info") +  def handle_ia_xauth(email, password): -    resp = requests.post(Config.IA_XAUTH_URI, -        params={'op': 'authenticate'}, +    resp = requests.post( +        Config.IA_XAUTH_URI, +        params={"op": "authenticate"},          json={ -            'version': '1', -            'email': email, -            'password': password, -            'access': Config.IA_XAUTH_CLIENT_ID, -            'secret': Config.IA_XAUTH_CLIENT_SECRET, -        }) -    if resp.status_code == 401 or (not resp.json().get('success')): +            "version": "1", +            "email": email, +            "password": password, +            "access": Config.IA_XAUTH_CLIENT_ID, +            "secret": Config.IA_XAUTH_CLIENT_SECRET, +        }, +    ) +    if resp.status_code == 401 or (not resp.json().get("success")):          try: -            flash("Internet Archive email/password didn't match: {}".format(resp.json()['values']['reason'])) +            flash( +                "Internet Archive email/password didn't match: {}".format( +                    resp.json()["values"]["reason"] +                ) +            )          except Exception:              app.log.warning("IA XAuth fail: {}".format(resp.content)) -        return render_template('auth_ia_login.html', email=email), resp.status_code +        return render_template("auth_ia_login.html", email=email), resp.status_code      elif resp.status_code != 200:          flash("Internet Archive login failed (internal error?)")          app.log.warning("IA XAuth fail: {}".format(resp.content)) -        return render_template('auth_ia_login.html', email=email), resp.status_code +        return render_template("auth_ia_login.html", email=email), resp.status_code      # Successful login; now fetch info... -    resp = requests.post(Config.IA_XAUTH_URI, -        params={'op': 'info'}, +    resp = requests.post( +        Config.IA_XAUTH_URI, +        params={"op": "info"},          json={ -            'version': '1', -            'email': email, -            'access': Config.IA_XAUTH_CLIENT_ID, -            'secret': Config.IA_XAUTH_CLIENT_SECRET, -        }) +            "version": "1", +            "email": email, +            "access": Config.IA_XAUTH_CLIENT_ID, +            "secret": Config.IA_XAUTH_CLIENT_SECRET, +        }, +    )      if resp.status_code != 200:          flash("Internet Archive login failed (internal error?)")          app.log.warning("IA XAuth fail: {}".format(resp.content)) -        return render_template('auth_ia_login.html', email=email), resp.status_code -    ia_info = resp.json()['values'] +        return render_template("auth_ia_login.html", email=email), resp.status_code +    ia_info = resp.json()["values"]      # and pass off "as if" we did OAuth successfully -    FakeOAuthRemote = namedtuple('FakeOAuthRemote', ['name', 'OAUTH_CONFIG']) -    remote = FakeOAuthRemote(name='archive', OAUTH_CONFIG={'api_base_url': Config.IA_XAUTH_URI}) +    FakeOAuthRemote = namedtuple("FakeOAuthRemote", ["name", "OAUTH_CONFIG"]) +    remote = FakeOAuthRemote(name="archive", OAUTH_CONFIG={"api_base_url": Config.IA_XAUTH_URI})      oauth_info = { -        'preferred_username': ia_info['itemname'].replace('@', ''), -        'iss': Config.IA_XAUTH_URI, -        'sub': ia_info['itemname'], +        "preferred_username": ia_info["itemname"].replace("@", ""), +        "iss": Config.IA_XAUTH_URI, +        "sub": ia_info["itemname"],      }      return handle_oauth(remote, None, oauth_info) +  def handle_wmoauth(username):      # pass off "as if" we did OAuth successfully -    FakeOAuthRemote = namedtuple('FakeOAuthRemote', ['name', 'OAUTH_CONFIG']) -    remote = FakeOAuthRemote(name='wikipedia', OAUTH_CONFIG={'api_base_url': "https://www.mediawiki.org/w"}) -    conservative_username = ''.join(filter(str.isalnum, username)) +    FakeOAuthRemote = namedtuple("FakeOAuthRemote", ["name", "OAUTH_CONFIG"]) +    remote = FakeOAuthRemote( +        name="wikipedia", OAUTH_CONFIG={"api_base_url": "https://www.mediawiki.org/w"} +    ) +    conservative_username = "".join(filter(str.isalnum, username))      oauth_info = { -        'preferred_username': conservative_username, -        'iss': "https://www.mediawiki.org/w", -        'sub': username, +        "preferred_username": conservative_username, +        "iss": "https://www.mediawiki.org/w", +        "sub": username,      }      return handle_oauth(remote, None, oauth_info) +  @login_manager.user_loader  def load_user(editor_id):      # looks for extra info in session, and updates the user object with that.      # If session isn't loaded/valid, should return None -    if (not session.get('editor')) or (not session.get('api_token')): +    if (not session.get("editor")) or (not session.get("api_token")):          return None -    editor = session['editor'] -    token = session['api_token'] +    editor = session["editor"] +    token = session["api_token"]      user = UserMixin()      user.id = editor_id      user.editor_id = editor_id -    user.username = editor['username'] -    user.is_admin = editor['is_admin'] +    user.username = editor["username"] +    user.is_admin = editor["is_admin"]      user.token = token      return user diff --git a/python/fatcat_web/cors.py b/python/fatcat_web/cors.py index cb2054b2..bb32f7c2 100644 --- a/python/fatcat_web/cors.py +++ b/python/fatcat_web/cors.py @@ -1,4 +1,3 @@ -  """  This snippet from: http://flask.pocoo.org/snippets/56/  "Posted by Armin Ronacher on 2011-07-14" @@ -10,15 +9,20 @@ from functools import update_wrapper  from flask import current_app, make_response, request -def crossdomain(origin=None, methods=None, headers=None, -                max_age=21600, attach_to_all=True, -                automatic_options=True): +def crossdomain( +    origin=None, +    methods=None, +    headers=None, +    max_age=21600, +    attach_to_all=True, +    automatic_options=True, +):      if methods is not None: -        methods = ', '.join(sorted(x.upper() for x in methods)) +        methods = ", ".join(sorted(x.upper() for x in methods))      if headers is not None and not isinstance(headers, str): -        headers = ', '.join(x.upper() for x in headers) +        headers = ", ".join(x.upper() for x in headers)      if not isinstance(origin, str): -        origin = ', '.join(origin) +        origin = ", ".join(origin)      if isinstance(max_age, timedelta):          max_age = max_age.total_seconds() @@ -27,26 +31,27 @@ def crossdomain(origin=None, methods=None, headers=None,              return methods          options_resp = current_app.make_default_options_response() -        return options_resp.headers['allow'] +        return options_resp.headers["allow"]      def decorator(f):          def wrapped_function(*args, **kwargs): -            if automatic_options and request.method == 'OPTIONS': +            if automatic_options and request.method == "OPTIONS":                  resp = current_app.make_default_options_response()              else:                  resp = make_response(f(*args, **kwargs)) -            if not attach_to_all and request.method != 'OPTIONS': +            if not attach_to_all and request.method != "OPTIONS":                  return resp              h = resp.headers -            h['Access-Control-Allow-Origin'] = origin -            h['Access-Control-Allow-Methods'] = get_methods() -            h['Access-Control-Max-Age'] = str(max_age) +            h["Access-Control-Allow-Origin"] = origin +            h["Access-Control-Allow-Methods"] = get_methods() +            h["Access-Control-Max-Age"] = str(max_age)              if headers is not None: -                h['Access-Control-Allow-Headers'] = headers +                h["Access-Control-Allow-Headers"] = headers              return resp          f.provide_automatic_options = False          return update_wrapper(wrapped_function, f) +      return decorator diff --git a/python/fatcat_web/editing_routes.py b/python/fatcat_web/editing_routes.py index 5a97dfc4..6dafd2f1 100644 --- a/python/fatcat_web/editing_routes.py +++ b/python/fatcat_web/editing_routes.py @@ -1,4 +1,3 @@ -  from typing import Optional  from fatcat_openapi_client import ( @@ -29,47 +28,53 @@ from fatcat_web.forms import (  ### Helper Methods ########################################################## -def generic_entity_create_from_toml(user_api, entity_type: str, editgroup_id: str, toml_str: str) -> EntityEdit: -    if entity_type == 'container': + +def generic_entity_create_from_toml( +    user_api, entity_type: str, editgroup_id: str, toml_str: str +) -> EntityEdit: +    if entity_type == "container":          entity = entity_from_toml(toml_str, ContainerEntity)          edit = user_api.create_container(editgroup_id, entity) -    elif entity_type == 'creator': +    elif entity_type == "creator":          entity = entity_from_toml(toml_str, CreatorEntity)          edit = user_api.create_creator(editgroup_id, entity) -    elif entity_type == 'file': +    elif entity_type == "file":          entity = entity_from_toml(toml_str, FileEntity)          edit = user_api.create_file(editgroup_id, entity) -    elif entity_type == 'fileset': +    elif entity_type == "fileset":          entity = entity_from_toml(toml_str, FilesetEntity)          edit = user_api.create_fileset(editgroup_id, entity) -    elif entity_type == 'webcapture': +    elif entity_type == "webcapture":          entity = entity_from_toml(toml_str, WebcaptureEntity)          edit = user_api.create_webcapture(editgroup_id, entity) -    elif entity_type == 'release': +    elif entity_type == "release":          entity = entity_from_toml(toml_str, ReleaseEntity)          edit = user_api.create_release(editgroup_id, entity) -    elif entity_type == 'work': +    elif entity_type == "work":          entity = entity_from_toml(toml_str, WorkEntity)          edit = user_api.create_work(editgroup_id, entity)      else:          raise NotImplementedError      return edit -def generic_entity_delete_edit(user_api, entity_type: str, editgroup_id: str, edit_id: str) -> None: + +def generic_entity_delete_edit( +    user_api, entity_type: str, editgroup_id: str, edit_id: str +) -> None:      try: -        if entity_type == 'container': +        if entity_type == "container":              user_api.delete_container_edit(editgroup_id, edit_id) -        elif entity_type == 'creator': +        elif entity_type == "creator":              user_api.delete_creator_edit(editgroup_id, edit_id) -        elif entity_type == 'file': +        elif entity_type == "file":              user_api.delete_file_edit(editgroup_id, edit_id) -        elif entity_type == 'fileset': +        elif entity_type == "fileset":              user_api.delete_fileset_edit(editgroup_id, edit_id) -        elif entity_type == 'webcapture': +        elif entity_type == "webcapture":              user_api.delete_webcapture_edit(editgroup_id, edit_id) -        elif entity_type == 'release': +        elif entity_type == "release":              user_api.delete_release_edit(editgroup_id, edit_id) -        elif entity_type == 'work': +        elif entity_type == "work":              user_api.delete_work_edit(editgroup_id, edit_id)          else:              raise NotImplementedError @@ -79,21 +84,24 @@ def generic_entity_delete_edit(user_api, entity_type: str, editgroup_id: str, ed          else:              raise ae -def generic_entity_delete_entity(user_api, entity_type: str, editgroup_id: str, entity_ident: str) -> None: + +def generic_entity_delete_entity( +    user_api, entity_type: str, editgroup_id: str, entity_ident: str +) -> None:      try: -        if entity_type == 'container': +        if entity_type == "container":              edit = user_api.delete_container(editgroup_id, entity_ident) -        elif entity_type == 'creator': +        elif entity_type == "creator":              edit = user_api.delete_creator(editgroup_id, entity_ident) -        elif entity_type == 'file': +        elif entity_type == "file":              edit = user_api.delete_file(editgroup_id, entity_ident) -        elif entity_type == 'fileset': +        elif entity_type == "fileset":              edit = user_api.delete_fileset(editgroup_id, entity_ident) -        elif entity_type == 'webcapture': +        elif entity_type == "webcapture":              edit = user_api.delete_webcapture(editgroup_id, entity_ident) -        elif entity_type == 'release': +        elif entity_type == "release":              edit = user_api.delete_release(editgroup_id, entity_ident) -        elif entity_type == 'work': +        elif entity_type == "work":              edit = user_api.delete_work(editgroup_id, entity_ident)          else:              raise NotImplementedError @@ -101,32 +109,36 @@ def generic_entity_delete_entity(user_api, entity_type: str, editgroup_id: str,          raise ae      return edit -def generic_entity_update_from_toml(user_api, entity_type: str, editgroup_id: str, existing_ident, toml_str: str) -> EntityEdit: -    if entity_type == 'container': + +def generic_entity_update_from_toml( +    user_api, entity_type: str, editgroup_id: str, existing_ident, toml_str: str +) -> EntityEdit: +    if entity_type == "container":          entity = entity_from_toml(toml_str, ContainerEntity)          edit = user_api.update_container(editgroup_id, existing_ident, entity) -    elif entity_type == 'creator': +    elif entity_type == "creator":          entity = entity_from_toml(toml_str, CreatorEntity)          edit = user_api.update_creator(editgroup_id, existing_ident, entity) -    elif entity_type == 'file': +    elif entity_type == "file":          entity = entity_from_toml(toml_str, FileEntity)          edit = user_api.update_file(editgroup_id, existing_ident, entity) -    elif entity_type == 'fileset': +    elif entity_type == "fileset":          entity = entity_from_toml(toml_str, FilesetEntity)          edit = user_api.update_fileset(editgroup_id, existing_ident, entity) -    elif entity_type == 'webcapture': +    elif entity_type == "webcapture":          entity = entity_from_toml(toml_str, WebcaptureEntity)          edit = user_api.update_webcapture(editgroup_id, existing_ident, entity) -    elif entity_type == 'release': +    elif entity_type == "release":          entity = entity_from_toml(toml_str, ReleaseEntity)          edit = user_api.update_release(editgroup_id, existing_ident, entity) -    elif entity_type == 'work': +    elif entity_type == "work":          entity = entity_from_toml(toml_str, WorkEntity)          edit = user_api.update_work(editgroup_id, existing_ident, entity)      else:          raise NotImplementedError      return edit +  def form_editgroup_get_or_create(api, edit_form):      """      This function expects a submitted, validated edit form @@ -147,13 +159,15 @@ def form_editgroup_get_or_create(api, edit_form):          # if no editgroup, create one from description          try:              eg = api.create_editgroup( -                Editgroup(description=edit_form.editgroup_description.data or None)) +                Editgroup(description=edit_form.editgroup_description.data or None) +            )          except ApiException as ae:              app.log.warning(ae)              raise ae          # set this session editgroup_id (TODO)      return eg +  def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template):      """ @@ -195,17 +209,19 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template      existing = None      existing_edit = None      if editgroup and existing_ident: -        existing, existing_edit = generic_get_editgroup_entity(editgroup, entity_type, existing_ident) +        existing, existing_edit = generic_get_editgroup_entity( +            editgroup, entity_type, existing_ident +        )      elif existing_ident:          existing = generic_get_entity(entity_type, existing_ident)      # parse form (if submitted)      status = 200 -    if entity_type == 'container': +    if entity_type == "container":          form = ContainerEntityForm() -    elif entity_type == 'file': +    elif entity_type == "file":          form = FileEntityForm() -    elif entity_type == 'release': +    elif entity_type == "release":          form = ReleaseEntityForm()      else:          raise NotImplementedError @@ -213,28 +229,32 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template      if form.is_submitted():          if form.validate_on_submit():              # API on behalf of user -            user_api = auth_api(session['api_token']) +            user_api = auth_api(session["api_token"])              if not editgroup:                  editgroup = form_editgroup_get_or_create(user_api, form)              if editgroup: -                if not existing_ident: # it's a create +                if not existing_ident:  # it's a create                      entity = form.to_entity()                      try: -                        if entity_type == 'container': +                        if entity_type == "container":                              edit = user_api.create_container(editgroup.editgroup_id, entity) -                        elif entity_type == 'file': +                        elif entity_type == "file":                              edit = user_api.create_file(editgroup.editgroup_id, entity) -                        elif entity_type == 'release': +                        elif entity_type == "release":                              edit = user_api.create_release(editgroup.editgroup_id, entity)                          else:                              raise NotImplementedError                      except ApiException as ae:                          app.log.warning(ae)                          raise ae -                    return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) -                else: # it's an update +                    return redirect( +                        "/editgroup/{}/{}/{}".format( +                            editgroup.editgroup_id, entity_type, edit.ident +                        ) +                    ) +                else:  # it's an update                      # all the tricky logic is in the update method                      form.update_entity(existing)                      # do we need to try to delete the current in-progress edit first? @@ -248,44 +268,61 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template                          # a "update pointer" edit                          existing.revision = None                          try: -                            generic_entity_delete_edit(user_api, entity_type, editgroup.editgroup_id, existing_edit.edit_id) +                            generic_entity_delete_edit( +                                user_api, +                                entity_type, +                                editgroup.editgroup_id, +                                existing_edit.edit_id, +                            )                          except ApiException as ae:                              if ae.status == 404:                                  pass                              else:                                  raise ae                      try: -                        if entity_type == 'container': -                            edit = user_api.update_container(editgroup.editgroup_id, existing.ident, existing) -                        elif entity_type == 'file': -                            edit = user_api.update_file(editgroup.editgroup_id, existing.ident, existing) -                        elif entity_type == 'release': -                            edit = user_api.update_release(editgroup.editgroup_id, existing.ident, existing) +                        if entity_type == "container": +                            edit = user_api.update_container( +                                editgroup.editgroup_id, existing.ident, existing +                            ) +                        elif entity_type == "file": +                            edit = user_api.update_file( +                                editgroup.editgroup_id, existing.ident, existing +                            ) +                        elif entity_type == "release": +                            edit = user_api.update_release( +                                editgroup.editgroup_id, existing.ident, existing +                            )                          else:                              raise NotImplementedError                      except ApiException as ae:                          app.log.warning(ae)                          raise ae -                    return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) +                    return redirect( +                        "/editgroup/{}/{}/{}".format( +                            editgroup.editgroup_id, entity_type, edit.ident +                        ) +                    )              else:                  status = 400          elif form.errors:              status = 400              app.log.info("form errors (did not validate): {}".format(form.errors)) -    else: # form is not submitted +    else:  # form is not submitted          if existing: -            if entity_type == 'container': +            if entity_type == "container":                  form = ContainerEntityForm.from_entity(existing) -            elif entity_type == 'file': +            elif entity_type == "file":                  form = FileEntityForm.from_entity(existing) -            elif entity_type == 'release': +            elif entity_type == "release":                  form = ReleaseEntityForm.from_entity(existing)              else:                  raise NotImplementedError -    editor_editgroups = api.get_editor_editgroups(session['editor']['editor_id'], limit=20) -    potential_editgroups = [e for e in editor_editgroups if e.changelog_index is None and e.submitted is None] +    editor_editgroups = api.get_editor_editgroups(session["editor"]["editor_id"], limit=20) +    potential_editgroups = [ +        e for e in editor_editgroups if e.changelog_index is None and e.submitted is None +    ]      if not form.is_submitted():          # default to most recent not submitted, fallback to "create new" @@ -293,9 +330,17 @@ def generic_entity_edit(editgroup_id, entity_type, existing_ident, edit_template          if potential_editgroups:              form.editgroup_id.data = potential_editgroups[0].editgroup_id -    return render_template(edit_template, form=form, -        existing_ident=existing_ident, editgroup=editgroup, -        potential_editgroups=potential_editgroups), status +    return ( +        render_template( +            edit_template, +            form=form, +            existing_ident=existing_ident, +            editgroup=editgroup, +            potential_editgroups=potential_editgroups, +        ), +        status, +    ) +  def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_template):      """ @@ -321,7 +366,9 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem      existing = None      existing_edit = None      if editgroup and existing_ident: -        existing, existing_edit = generic_get_editgroup_entity(editgroup, entity_type, existing_ident) +        existing, existing_edit = generic_get_editgroup_entity( +            editgroup, entity_type, existing_ident +        )      elif existing_ident:          existing = generic_get_entity(entity_type, existing_ident) @@ -332,15 +379,17 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem      if form.is_submitted():          if form.validate_on_submit():              # API on behalf of user -            user_api = auth_api(session['api_token']) +            user_api = auth_api(session["api_token"])              if not editgroup:                  editgroup = form_editgroup_get_or_create(user_api, form)              if editgroup: -                if not existing_ident: # it's a create +                if not existing_ident:  # it's a create                      try: -                        edit = generic_entity_create_from_toml(user_api, entity_type, editgroup.editgroup_id, form.toml.data) +                        edit = generic_entity_create_from_toml( +                            user_api, entity_type, editgroup.editgroup_id, form.toml.data +                        )                      except ValueError as ve:                          form.toml.errors = [ve]                          status = 400 @@ -348,8 +397,12 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem                          app.log.warning(ae)                          raise ae                      if status == 200: -                        return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) -                else: # it's an update +                        return redirect( +                            "/editgroup/{}/{}/{}".format( +                                editgroup.editgroup_id, entity_type, edit.ident +                            ) +                        ) +                else:  # it's an update                      # TODO: some danger of wiping database state here is                      # "updated edit" causes, eg, a 4xx error. Better to allow                      # this in the API itself. For now, form validation *should* @@ -359,9 +412,17 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem                          # need to clear revision on object or this becomes just                          # a "update pointer" edit                          existing.revision = None -                        generic_entity_delete_edit(user_api, entity_type, editgroup.editgroup_id, existing_edit.edit_id) +                        generic_entity_delete_edit( +                            user_api, entity_type, editgroup.editgroup_id, existing_edit.edit_id +                        )                      try: -                        edit = generic_entity_update_from_toml(user_api, entity_type, editgroup.editgroup_id, existing.ident, form.toml.data) +                        edit = generic_entity_update_from_toml( +                            user_api, +                            entity_type, +                            editgroup.editgroup_id, +                            existing.ident, +                            form.toml.data, +                        )                      except ValueError as ve:                          form.toml.errors = [ve]                          status = 400 @@ -369,19 +430,25 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem                          app.log.warning(ae)                          raise ae                      if status == 200: -                        return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) +                        return redirect( +                            "/editgroup/{}/{}/{}".format( +                                editgroup.editgroup_id, entity_type, edit.ident +                            ) +                        )              else:                  status = 400          elif form.errors:              status = 400              app.log.info("form errors (did not validate): {}".format(form.errors)) -    else: # form is not submitted +    else:  # form is not submitted          if existing:              form = EntityTomlForm.from_entity(existing) -    editor_editgroups = api.get_editor_editgroups(session['editor']['editor_id'], limit=20) -    potential_editgroups = [e for e in editor_editgroups if e.changelog_index is None and e.submitted is None] +    editor_editgroups = api.get_editor_editgroups(session["editor"]["editor_id"], limit=20) +    potential_editgroups = [ +        e for e in editor_editgroups if e.changelog_index is None and e.submitted is None +    ]      if not form.is_submitted():          # default to most recent not submitted, fallback to "create new" @@ -389,9 +456,18 @@ def generic_entity_toml_edit(editgroup_id, entity_type, existing_ident, edit_tem          if potential_editgroups:              form.editgroup_id.data = potential_editgroups[0].editgroup_id -    return render_template(edit_template, form=form, entity_type=entity_type, -        existing_ident=existing_ident, editgroup=editgroup, -        potential_editgroups=potential_editgroups), status +    return ( +        render_template( +            edit_template, +            form=form, +            entity_type=entity_type, +            existing_ident=existing_ident, +            editgroup=editgroup, +            potential_editgroups=potential_editgroups, +        ), +        status, +    ) +  def generic_entity_delete(editgroup_id: Optional[str], entity_type: str, existing_ident: str):      """ @@ -418,7 +494,9 @@ def generic_entity_delete(editgroup_id: Optional[str], entity_type: str, existin      existing = None      existing_edit = None      if editgroup and existing_ident: -        existing, existing_edit = generic_get_editgroup_entity(editgroup, entity_type, existing_ident) +        existing, existing_edit = generic_get_editgroup_entity( +            editgroup, entity_type, existing_ident +        )      elif existing_ident:          existing = generic_get_entity(entity_type, existing_ident) @@ -429,7 +507,7 @@ def generic_entity_delete(editgroup_id: Optional[str], entity_type: str, existin      if form.is_submitted():          if form.validate_on_submit():              # API on behalf of user -            user_api = auth_api(session['api_token']) +            user_api = auth_api(session["api_token"])              if not editgroup:                  editgroup = form_editgroup_get_or_create(user_api, form) @@ -443,26 +521,36 @@ def generic_entity_delete(editgroup_id: Optional[str], entity_type: str, existin                      # need to clear revision on object or this becomes just                      # a "update pointer" edit                      existing.revision = None -                    generic_entity_delete_edit(user_api, entity_type, editgroup.editgroup_id, existing_edit.edit_id) +                    generic_entity_delete_edit( +                        user_api, entity_type, editgroup.editgroup_id, existing_edit.edit_id +                    )                  try: -                    edit = generic_entity_delete_entity(user_api, entity_type, editgroup.editgroup_id, existing.ident) +                    edit = generic_entity_delete_entity( +                        user_api, entity_type, editgroup.editgroup_id, existing.ident +                    )                  except ApiException as ae:                      app.log.warning(ae)                      raise ae                  if status == 200: -                    return redirect('/editgroup/{}/{}/{}'.format(editgroup.editgroup_id, entity_type, edit.ident)) +                    return redirect( +                        "/editgroup/{}/{}/{}".format( +                            editgroup.editgroup_id, entity_type, edit.ident +                        ) +                    )              else:                  status = 400          elif form.errors:              status = 400              app.log.info("form errors (did not validate): {}".format(form.errors)) -    else: # form is not submitted +    else:  # form is not submitted          if existing:              form = EntityTomlForm.from_entity(existing) -    editor_editgroups = api.get_editor_editgroups(session['editor']['editor_id'], limit=20) -    potential_editgroups = [e for e in editor_editgroups if e.changelog_index is None and e.submitted is None] +    editor_editgroups = api.get_editor_editgroups(session["editor"]["editor_id"], limit=20) +    potential_editgroups = [ +        e for e in editor_editgroups if e.changelog_index is None and e.submitted is None +    ]      if not form.is_submitted():          # default to most recent not submitted, fallback to "create new" @@ -470,9 +558,18 @@ def generic_entity_delete(editgroup_id: Optional[str], entity_type: str, existin          if potential_editgroups:              form.editgroup_id.data = potential_editgroups[0].editgroup_id -    return render_template("entity_delete.html", form=form, entity_type=entity_type, -        existing_ident=existing_ident, editgroup=editgroup, -        potential_editgroups=potential_editgroups), status +    return ( +        render_template( +            "entity_delete.html", +            form=form, +            entity_type=entity_type, +            existing_ident=existing_ident, +            editgroup=editgroup, +            potential_editgroups=potential_editgroups, +        ), +        status, +    ) +  def generic_edit_delete(editgroup_id, entity_type, edit_id):      # fetch editgroup (if set) or 404 @@ -489,7 +586,7 @@ def generic_edit_delete(editgroup_id, entity_type, edit_id):              abort(400)      # API on behalf of user -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      # do the deletion      generic_entity_delete_edit(user_api, entity_type, editgroup.editgroup_id, edit_id) @@ -498,317 +595,382 @@ def generic_edit_delete(editgroup_id, entity_type, edit_id):  ### Views ################################################################### -@app.route('/container/create', methods=['GET', 'POST']) + +@app.route("/container/create", methods=["GET", "POST"])  @login_required  def container_create_view(): -    return generic_entity_edit(None, 'container', None, 'container_create.html') +    return generic_entity_edit(None, "container", None, "container_create.html") + -@app.route('/container/<ident>/edit', methods=['GET', 'POST']) +@app.route("/container/<ident>/edit", methods=["GET", "POST"])  @login_required  def container_edit_view(ident): -    return generic_entity_edit(None, 'container', ident, 'container_edit.html') +    return generic_entity_edit(None, "container", ident, "container_edit.html") + -@app.route('/container/<ident>/delete', methods=['GET', 'POST']) +@app.route("/container/<ident>/delete", methods=["GET", "POST"])  @login_required  def container_delete_view(ident): -    return generic_entity_delete(None, 'container', ident) +    return generic_entity_delete(None, "container", ident) -@app.route('/editgroup/<editgroup_id>/container/<ident>/edit', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/container/<ident>/edit", methods=["GET", "POST"])  @login_required  def container_editgroup_edit_view(editgroup_id, ident): -    return generic_entity_edit(editgroup_id, 'container', ident, 'container_edit.html') +    return generic_entity_edit(editgroup_id, "container", ident, "container_edit.html") + -@app.route('/editgroup/<editgroup_id>/container/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/container/<ident>/delete", methods=["GET", "POST"])  @login_required  def container_editgroup_delete_view(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'container', ident) +    return generic_entity_delete(editgroup_id, "container", ident) -@app.route('/editgroup/<editgroup_id>/container/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/container/edit/<edit_id>/delete", methods=["POST"])  @login_required  def container_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'container', edit_id) +    return generic_edit_delete(editgroup_id, "container", edit_id) + -@app.route('/creator/<ident>/delete', methods=['GET', 'POST']) +@app.route("/creator/<ident>/delete", methods=["GET", "POST"])  @login_required  def creator_delete_view(ident): -    return generic_entity_delete(None, 'creator', ident) +    return generic_entity_delete(None, "creator", ident) -@app.route('/editgroup/<editgroup_id>/creator/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/creator/edit/<edit_id>/delete", methods=["POST"])  def creator_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'creator', edit_id) +    return generic_edit_delete(editgroup_id, "creator", edit_id) + -@app.route('/editgroup/<editgroup_id>/creator/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/creator/<ident>/delete", methods=["GET", "POST"])  @login_required  def creator_editgroup_delete(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'creator', ident) +    return generic_entity_delete(editgroup_id, "creator", ident) -@app.route('/file/create', methods=['GET', 'POST']) + +@app.route("/file/create", methods=["GET", "POST"])  @login_required  def file_create_view(): -    return generic_entity_edit(None, 'file', None, 'file_create.html') +    return generic_entity_edit(None, "file", None, "file_create.html") + -@app.route('/file/<ident>/edit', methods=['GET', 'POST']) +@app.route("/file/<ident>/edit", methods=["GET", "POST"])  @login_required  def file_edit_view(ident): -    return generic_entity_edit(None, 'file', ident, 'file_edit.html') +    return generic_entity_edit(None, "file", ident, "file_edit.html") + -@app.route('/file/<ident>/delete', methods=['GET', 'POST']) +@app.route("/file/<ident>/delete", methods=["GET", "POST"])  @login_required  def file_delete_view(ident): -    return generic_entity_delete(None, 'file', ident) +    return generic_entity_delete(None, "file", ident) -@app.route('/editgroup/<editgroup_id>/file/<ident>/edit', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/file/<ident>/edit", methods=["GET", "POST"])  @login_required  def file_editgroup_edit_view(editgroup_id, ident): -    return generic_entity_edit(editgroup_id, 'file', ident, 'file_edit.html') +    return generic_entity_edit(editgroup_id, "file", ident, "file_edit.html") + -@app.route('/editgroup/<editgroup_id>/file/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/file/<ident>/delete", methods=["GET", "POST"])  @login_required  def file_editgroup_delete_view(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'file', ident) +    return generic_entity_delete(editgroup_id, "file", ident) -@app.route('/editgroup/<editgroup_id>/file/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/file/edit/<edit_id>/delete", methods=["POST"])  @login_required  def file_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'file', edit_id) +    return generic_edit_delete(editgroup_id, "file", edit_id) + -@app.route('/fileset/<ident>/delete', methods=['GET', 'POST']) +@app.route("/fileset/<ident>/delete", methods=["GET", "POST"])  @login_required  def fileset_delete_view(ident): -    return generic_entity_delete(None, 'fileset', ident) +    return generic_entity_delete(None, "fileset", ident) + -@app.route('/editgroup/<editgroup_id>/fileset/edit/<edit_id>/delete', methods=['POST']) +@app.route("/editgroup/<editgroup_id>/fileset/edit/<edit_id>/delete", methods=["POST"])  def fileset_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'fileset', edit_id) +    return generic_edit_delete(editgroup_id, "fileset", edit_id) -@app.route('/editgroup/<editgroup_id>/fileset/<ident>/delete', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/fileset/<ident>/delete", methods=["GET", "POST"])  @login_required  def fileset_editgroup_delete(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'fileset', ident) +    return generic_entity_delete(editgroup_id, "fileset", ident) + -@app.route('/webcapture/<ident>/delete', methods=['GET', 'POST']) +@app.route("/webcapture/<ident>/delete", methods=["GET", "POST"])  @login_required  def webcapture_delete_view(ident): -    return generic_entity_delete(None, 'webcapture', ident) +    return generic_entity_delete(None, "webcapture", ident) -@app.route('/editgroup/<editgroup_id>/webcapture/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/webcapture/edit/<edit_id>/delete", methods=["POST"])  def webcapture_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'webcapture', edit_id) +    return generic_edit_delete(editgroup_id, "webcapture", edit_id) + -@app.route('/editgroup/<editgroup_id>/webcapture/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/webcapture/<ident>/delete", methods=["GET", "POST"])  @login_required  def webcapture_editgroup_delete(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'webcapture', ident) +    return generic_entity_delete(editgroup_id, "webcapture", ident) + -@app.route('/release/create', methods=['GET', 'POST']) +@app.route("/release/create", methods=["GET", "POST"])  @login_required  def release_create_view(): -    return generic_entity_edit(None, 'release', None, 'release_create.html') +    return generic_entity_edit(None, "release", None, "release_create.html") -@app.route('/release/<ident>/edit', methods=['GET', 'POST']) + +@app.route("/release/<ident>/edit", methods=["GET", "POST"])  @login_required  def release_edit_view(ident): -    return generic_entity_edit(None, 'release', ident, 'release_edit.html') +    return generic_entity_edit(None, "release", ident, "release_edit.html") + -@app.route('/release/<ident>/delete', methods=['GET', 'POST']) +@app.route("/release/<ident>/delete", methods=["GET", "POST"])  @login_required  def release_delete_view(ident): -    return generic_entity_delete(None, 'release', ident) +    return generic_entity_delete(None, "release", ident) -@app.route('/editgroup/<editgroup_id>/release/<ident>/edit', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/release/<ident>/edit", methods=["GET", "POST"])  @login_required  def release_editgroup_edit(editgroup_id, ident): -    return generic_entity_edit(editgroup_id, 'release', ident, 'release_edit.html') +    return generic_entity_edit(editgroup_id, "release", ident, "release_edit.html") + -@app.route('/editgroup/<editgroup_id>/release/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/release/<ident>/delete", methods=["GET", "POST"])  @login_required  def release_editgroup_delete(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'release', ident) +    return generic_entity_delete(editgroup_id, "release", ident) -@app.route('/editgroup/<editgroup_id>/release/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/release/edit/<edit_id>/delete", methods=["POST"])  @login_required  def release_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'release', edit_id) +    return generic_edit_delete(editgroup_id, "release", edit_id) + -@app.route('/work/<ident>/delete', methods=['GET', 'POST']) +@app.route("/work/<ident>/delete", methods=["GET", "POST"])  @login_required  def work_delete_view(ident): -    return generic_entity_delete(None, 'work', ident) +    return generic_entity_delete(None, "work", ident) -@app.route('/editgroup/<editgroup_id>/work/edit/<edit_id>/delete', methods=['POST']) + +@app.route("/editgroup/<editgroup_id>/work/edit/<edit_id>/delete", methods=["POST"])  def work_edit_delete(editgroup_id, edit_id): -    return generic_edit_delete(editgroup_id, 'work', edit_id) +    return generic_edit_delete(editgroup_id, "work", edit_id) + -@app.route('/editgroup/<editgroup_id>/work/<ident>/delete', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/work/<ident>/delete", methods=["GET", "POST"])  @login_required  def work_editgroup_delete(editgroup_id, ident): -    return generic_entity_delete(editgroup_id, 'work', ident) +    return generic_entity_delete(editgroup_id, "work", ident) +  ### TOML Views ############################################################## -@app.route('/container/create/toml', methods=['GET', 'POST']) + +@app.route("/container/create/toml", methods=["GET", "POST"])  @login_required  def container_create_toml_view(): -    return generic_entity_toml_edit(None, 'container', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "container", None, "entity_create_toml.html") + -@app.route('/container/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/container/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def container_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'container', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "container", ident, "entity_edit_toml.html") -@app.route('/editgroup/<editgroup_id>/container/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/container/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def container_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'container', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "container", ident, "entity_edit_toml.html") + -@app.route('/creator/create/toml', methods=['GET', 'POST']) +@app.route("/creator/create/toml", methods=["GET", "POST"])  @login_required  def creator_create_toml_view(): -    return generic_entity_toml_edit(None, 'creator', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "creator", None, "entity_create_toml.html") + -@app.route('/creator/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/creator/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def creator_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'creator', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "creator", ident, "entity_edit_toml.html") -@app.route('/editgroup/<editgroup_id>/creator/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/creator/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def creator_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'creator', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "creator", ident, "entity_edit_toml.html") + -@app.route('/file/create/toml', methods=['GET', 'POST']) +@app.route("/file/create/toml", methods=["GET", "POST"])  @login_required  def file_create_toml_view(): -    return generic_entity_toml_edit(None, 'file', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "file", None, "entity_create_toml.html") -@app.route('/file/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/file/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def file_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'file', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "file", ident, "entity_edit_toml.html") + -@app.route('/editgroup/<editgroup_id>/file/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/file/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def file_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'file', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "file", ident, "entity_edit_toml.html") + -@app.route('/fileset/create/toml', methods=['GET', 'POST']) +@app.route("/fileset/create/toml", methods=["GET", "POST"])  @login_required  def fileset_create_toml_view(): -    return generic_entity_toml_edit(None, 'fileset', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "fileset", None, "entity_create_toml.html") -@app.route('/fileset/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/fileset/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def fileset_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'fileset', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "fileset", ident, "entity_edit_toml.html") + -@app.route('/editgroup/<editgroup_id>/fileset/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/fileset/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def fileset_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'fileset', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "fileset", ident, "entity_edit_toml.html") -@app.route('/webcapture/create/toml', methods=['GET', 'POST']) + +@app.route("/webcapture/create/toml", methods=["GET", "POST"])  @login_required  def webcapture_create_toml_view(): -    return generic_entity_toml_edit(None, 'webcapture', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "webcapture", None, "entity_create_toml.html") + -@app.route('/webcapture/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/webcapture/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def webcapture_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'webcapture', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "webcapture", ident, "entity_edit_toml.html") -@app.route('/editgroup/<editgroup_id>/webcapture/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/webcapture/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def webcapture_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'webcapture', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "webcapture", ident, "entity_edit_toml.html") + -@app.route('/release/create/toml', methods=['GET', 'POST']) +@app.route("/release/create/toml", methods=["GET", "POST"])  @login_required  def release_create_toml_view(): -    return generic_entity_toml_edit(None, 'release', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "release", None, "entity_create_toml.html") -@app.route('/release/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/release/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def release_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'release', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "release", ident, "entity_edit_toml.html") + -@app.route('/editgroup/<editgroup_id>/release/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/release/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def release_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'release', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "release", ident, "entity_edit_toml.html") + -@app.route('/work/create/toml', methods=['GET', 'POST']) +@app.route("/work/create/toml", methods=["GET", "POST"])  @login_required  def work_create_toml_view(): -    return generic_entity_toml_edit(None, 'work', None, 'entity_create_toml.html') +    return generic_entity_toml_edit(None, "work", None, "entity_create_toml.html") -@app.route('/work/<ident>/edit/toml', methods=['GET', 'POST']) + +@app.route("/work/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def work_edit_toml_view(ident): -    return generic_entity_toml_edit(None, 'work', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(None, "work", ident, "entity_edit_toml.html") + -@app.route('/editgroup/<editgroup_id>/work/<ident>/edit/toml', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/work/<ident>/edit/toml", methods=["GET", "POST"])  @login_required  def work_editgroup_edit_toml(editgroup_id, ident): -    return generic_entity_toml_edit(editgroup_id, 'work', ident, 'entity_edit_toml.html') +    return generic_entity_toml_edit(editgroup_id, "work", ident, "entity_edit_toml.html") +  ### TOML-Only Editing Redirects ################################################ -@app.route('/creator/create', methods=['GET']) + +@app.route("/creator/create", methods=["GET"])  @login_required  def creator_create_view(): -    return redirect('/creator/create/toml') +    return redirect("/creator/create/toml") -@app.route('/creator/<ident>/edit', methods=['GET']) + +@app.route("/creator/<ident>/edit", methods=["GET"])  @login_required  def creator_edit_view(ident): -    return redirect(f'/creator/{ident}/edit/toml') +    return redirect(f"/creator/{ident}/edit/toml") + -@app.route('/editgroup/<editgroup_id>/creator/<ident>/edit', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/creator/<ident>/edit", methods=["GET", "POST"])  @login_required  def creator_editgroup_edit(editgroup_id, ident): -    return redirect(f'/editgroup/{editgroup_id}/creator/{ident}/edit/toml') +    return redirect(f"/editgroup/{editgroup_id}/creator/{ident}/edit/toml") -@app.route('/fileset/create', methods=['GET']) + +@app.route("/fileset/create", methods=["GET"])  @login_required  def fileset_create_view(): -    return redirect('/fileset/create/toml') +    return redirect("/fileset/create/toml") + -@app.route('/fileset/<ident>/edit', methods=['GET']) +@app.route("/fileset/<ident>/edit", methods=["GET"])  @login_required  def fileset_edit_view(ident): -    return redirect(f'/fileset/{ident}/edit/toml') +    return redirect(f"/fileset/{ident}/edit/toml") + -@app.route('/editgroup/<editgroup_id>/fileset/<ident>/edit', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/fileset/<ident>/edit", methods=["GET", "POST"])  @login_required  def fileset_editgroup_edit(editgroup_id, ident): -    return redirect(f'/editgroup/{editgroup_id}/fileset/{ident}/edit/toml') +    return redirect(f"/editgroup/{editgroup_id}/fileset/{ident}/edit/toml") -@app.route('/webcapture/create', methods=['GET']) + +@app.route("/webcapture/create", methods=["GET"])  @login_required  def webcapture_create_view(): -    return redirect('/webcapture/create/toml') +    return redirect("/webcapture/create/toml") + -@app.route('/webcapture/<ident>/edit', methods=['GET']) +@app.route("/webcapture/<ident>/edit", methods=["GET"])  @login_required  def webcapture_edit_view(ident): -    return redirect(f'/webcapture/{ident}/edit/toml') +    return redirect(f"/webcapture/{ident}/edit/toml") -@app.route('/editgroup/<editgroup_id>/webcapture/<ident>/edit', methods=['GET', 'POST']) + +@app.route("/editgroup/<editgroup_id>/webcapture/<ident>/edit", methods=["GET", "POST"])  @login_required  def webcapture_editgroup_edit(editgroup_id, ident): -    return redirect(f'/editgroup/{editgroup_id}/webcapture/{ident}/edit/toml') +    return redirect(f"/editgroup/{editgroup_id}/webcapture/{ident}/edit/toml") + -@app.route('/work/create', methods=['GET']) +@app.route("/work/create", methods=["GET"])  @login_required  def work_create_view(): -    return redirect('/work/create/toml') +    return redirect("/work/create/toml") -@app.route('/work/<ident>/edit', methods=['GET']) + +@app.route("/work/<ident>/edit", methods=["GET"])  @login_required  def work_edit_view(ident): -    return redirect(f'/work/{ident}/edit/toml') +    return redirect(f"/work/{ident}/edit/toml") + -@app.route('/editgroup/<editgroup_id>/work/<ident>/edit', methods=['GET', 'POST']) +@app.route("/editgroup/<editgroup_id>/work/<ident>/edit", methods=["GET", "POST"])  @login_required  def work_editgroup_edit(editgroup_id, ident): -    return redirect(f'/editgroup/{editgroup_id}/work/{ident}/edit/toml') +    return redirect(f"/editgroup/{editgroup_id}/work/{ident}/edit/toml") diff --git a/python/fatcat_web/entity_helpers.py b/python/fatcat_web/entity_helpers.py index 5522f3b5..dbe11cb4 100644 --- a/python/fatcat_web/entity_helpers.py +++ b/python/fatcat_web/entity_helpers.py @@ -1,4 +1,3 @@ -  from fatcat_openapi_client import (      ContainerEntity,      CreatorEntity, @@ -22,41 +21,46 @@ from fatcat_web.hacks import strip_extlink_xml, wayback_suffix  def enrich_container_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      if entity.state == "active":          entity._es = container_to_elasticsearch(entity, force_bool=False)      return entity +  def enrich_creator_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      entity._releases = None -    if entity.state in ('active', 'wip'): +    if entity.state in ("active", "wip"):          entity._releases = api.get_creator_releases(entity.ident)      return entity +  def enrich_file_entity(entity):      if entity.state == "active":          entity._es = file_to_elasticsearch(entity)      return entity +  def enrich_fileset_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      entity._total_size = None      if entity.manifest is not None:          entity._total_size = sum([f.size for f in entity.manifest]) or 0      return entity +  def enrich_webcapture_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      entity._wayback_suffix = wayback_suffix(entity)      return entity +  def enrich_release_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      if entity.state == "active":          entity._es = release_to_elasticsearch(entity, force_bool=False) @@ -64,8 +68,9 @@ def enrich_release_entity(entity):          entity.container._es = container_to_elasticsearch(entity.container, force_bool=False)      if entity.files:          # remove shadows-only files with no URLs -        entity.files = [f for f in entity.files -            if not (f.extra and f.extra.get('shadows') and not f.urls)] +        entity.files = [ +            f for f in entity.files if not (f.extra and f.extra.get("shadows") and not f.urls) +        ]      if entity.filesets:          for fs in entity.filesets:              fs._total_size = sum([f.size for f in fs.manifest]) @@ -79,60 +84,74 @@ def enrich_release_entity(entity):          # xlink:href="http://lockss.org/"          # xlink:type="simple">http://lockss.org/</ext-link>. Accessed: 2014          # November 1. -        if ref.extra and ref.extra.get('unstructured'): -            ref.extra['unstructured'] = strip_extlink_xml(ref.extra['unstructured']) +        if ref.extra and ref.extra.get("unstructured"): +            ref.extra["unstructured"] = strip_extlink_xml(ref.extra["unstructured"])      # for backwards compatability, copy extra['subtitle'] to subtitle -    if not entity.subtitle and entity.extra and entity.extra.get('subtitle'): -        if isinstance(entity.extra['subtitle'], str): -            entity.subtitle = entity.extra['subtitle'] -        elif isinstance(entity.extra['subtitle'], list): -            entity.subtitle = entity.extra['subtitle'][0] or None +    if not entity.subtitle and entity.extra and entity.extra.get("subtitle"): +        if isinstance(entity.extra["subtitle"], str): +            entity.subtitle = entity.extra["subtitle"] +        elif isinstance(entity.extra["subtitle"], list): +            entity.subtitle = entity.extra["subtitle"][0] or None      # author list to display; ensure it's sorted by index (any othors with      # index=None go to end of list) -    authors = [c for c in entity.contribs if -        c.role in ('author', None) and -        (c.surname or c.raw_name or (c.creator and c.creator.surname)) +    authors = [ +        c +        for c in entity.contribs +        if c.role in ("author", None) +        and (c.surname or c.raw_name or (c.creator and c.creator.surname))      ]      entity._authors = sorted(authors, key=lambda c: (c.index is None and 99999999) or c.index)      # need authors, title for citeproc to work      entity._can_citeproc = bool(entity._authors) and bool(entity.title)      if entity.abstracts:          # hack to show plain text instead of latex abstracts -        if 'latex' in entity.abstracts[0].mimetype: +        if "latex" in entity.abstracts[0].mimetype:              entity.abstracts.reverse()          # hack to (partially) clean up common JATS abstract display case -        if entity.abstracts[0].mimetype == 'application/xml+jats': -            for tag in ('p', 'jats', 'jats:p'): -                entity.abstracts[0].content = entity.abstracts[0].content.replace('<{}>'.format(tag), '') -                entity.abstracts[0].content = entity.abstracts[0].content.replace('</{}>'.format(tag), '') +        if entity.abstracts[0].mimetype == "application/xml+jats": +            for tag in ("p", "jats", "jats:p"): +                entity.abstracts[0].content = entity.abstracts[0].content.replace( +                    "<{}>".format(tag), "" +                ) +                entity.abstracts[0].content = entity.abstracts[0].content.replace( +                    "</{}>".format(tag), "" +                )                  # ugh, double encoding happens -                entity.abstracts[0].content = entity.abstracts[0].content.replace('</{}>'.format(tag), '') -                entity.abstracts[0].content = entity.abstracts[0].content.replace('<{}>'.format(tag), '') +                entity.abstracts[0].content = entity.abstracts[0].content.replace( +                    "</{}>".format(tag), "" +                ) +                entity.abstracts[0].content = entity.abstracts[0].content.replace( +                    "<{}>".format(tag), "" +                )      return entity +  def enrich_work_entity(entity): -    if entity.state in ('redirect', 'deleted'): +    if entity.state in ("redirect", "deleted"):          return entity      entity._releases = None -    if entity.state in ('active', 'wip'): +    if entity.state in ("active", "wip"):          entity._releases = api.get_work_releases(entity.ident)      return entity +  def generic_get_entity(entity_type, ident):      try: -        if entity_type == 'container': +        if entity_type == "container":              return enrich_container_entity(api.get_container(ident)) -        elif entity_type == 'creator': +        elif entity_type == "creator":              return enrich_creator_entity(api.get_creator(ident)) -        elif entity_type == 'file': +        elif entity_type == "file":              return enrich_file_entity(api.get_file(ident, expand="releases")) -        elif entity_type == 'fileset': +        elif entity_type == "fileset":              return enrich_fileset_entity(api.get_fileset(ident, expand="releases")) -        elif entity_type == 'webcapture': +        elif entity_type == "webcapture":              return enrich_webcapture_entity(api.get_webcapture(ident, expand="releases")) -        elif entity_type == 'release': -            return enrich_release_entity(api.get_release(ident, expand="container,creators,files,filesets,webcaptures")) -        elif entity_type == 'work': +        elif entity_type == "release": +            return enrich_release_entity( +                api.get_release(ident, expand="container,creators,files,filesets,webcaptures") +            ) +        elif entity_type == "work":              return enrich_work_entity(api.get_work(ident))          else:              raise NotImplementedError @@ -141,21 +160,28 @@ def generic_get_entity(entity_type, ident):      except ApiValueError:          abort(400) +  def generic_get_entity_revision(entity_type, revision_id):      try: -        if entity_type == 'container': +        if entity_type == "container":              return enrich_container_entity(api.get_container_revision(revision_id)) -        elif entity_type == 'creator': +        elif entity_type == "creator":              return enrich_creator_entity(api.get_creator_revision(revision_id)) -        elif entity_type == 'file': +        elif entity_type == "file":              return enrich_file_entity(api.get_file_revision(revision_id, expand="releases")) -        elif entity_type == 'fileset': -            return enrich_fileset_entity(api.get_fileset_revision(revision_id, expand="releases")) -        elif entity_type == 'webcapture': -            return enrich_webcapture_entity(api.get_webcapture_revision(revision_id, expand="releases")) -        elif entity_type == 'release': -            return enrich_release_entity(api.get_release_revision(revision_id, expand="container")) -        elif entity_type == 'work': +        elif entity_type == "fileset": +            return enrich_fileset_entity( +                api.get_fileset_revision(revision_id, expand="releases") +            ) +        elif entity_type == "webcapture": +            return enrich_webcapture_entity( +                api.get_webcapture_revision(revision_id, expand="releases") +            ) +        elif entity_type == "release": +            return enrich_release_entity( +                api.get_release_revision(revision_id, expand="container") +            ) +        elif entity_type == "work":              return enrich_work_entity(api.get_work_revision(revision_id))          else:              raise NotImplementedError @@ -164,40 +190,42 @@ def generic_get_entity_revision(entity_type, revision_id):      except ApiValueError:          abort(400) +  def generic_deleted_entity(entity_type, ident): -    if entity_type == 'container': +    if entity_type == "container":          entity = ContainerEntity() -    elif entity_type == 'creator': +    elif entity_type == "creator":          entity = CreatorEntity() -    elif entity_type == 'file': +    elif entity_type == "file":          entity = FileEntity() -    elif entity_type == 'fileset': +    elif entity_type == "fileset":          entity = FilesetEntity() -    elif entity_type == 'webcapture': +    elif entity_type == "webcapture":          entity = WebcaptureEntity() -    elif entity_type == 'release': +    elif entity_type == "release":          entity = ReleaseEntity(ext_ids=ReleaseExtIds()) -    elif entity_type == 'work': +    elif entity_type == "work":          entity = WorkEntity()      else:          raise NotImplementedError      entity.ident = ident      return entity +  def generic_get_editgroup_entity(editgroup, entity_type, ident): -    if entity_type == 'container': +    if entity_type == "container":          edits = editgroup.edits.containers -    elif entity_type == 'creator': +    elif entity_type == "creator":          edits = editgroup.edits.creators -    elif entity_type == 'file': +    elif entity_type == "file":          edits = editgroup.edits.files -    elif entity_type == 'fileset': +    elif entity_type == "fileset":          edits = editgroup.edits.filesets -    elif entity_type == 'webcapture': +    elif entity_type == "webcapture":          edits = editgroup.edits.webcaptures -    elif entity_type == 'release': +    elif entity_type == "release":          edits = editgroup.edits.releases -    elif entity_type == 'work': +    elif entity_type == "work":          edits = editgroup.edits.works      else:          raise NotImplementedError diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py index b432ac16..25bfbb90 100644 --- a/python/fatcat_web/forms.py +++ b/python/fatcat_web/forms.py @@ -1,4 +1,3 @@ -  """  Note: in thoery could use, eg, https://github.com/christabor/swagger_wtforms,  but can't find one that is actually maintained. @@ -32,84 +31,99 @@ from wtforms import (  from fatcat_tools.transforms import entity_to_toml  release_type_options = [ -    ('', 'Unknown (blank)'), -    ('article-journal', 'Journal Article'), -    ('paper-conference', 'Conference Proceeding'), -    ('article', 'Article (non-journal)'), -    ('book', 'Book'), -    ('chapter', 'Book Chapter'), -    ('dataset', 'Dataset'), -    ('stub', 'Invalid/Stub'), +    ("", "Unknown (blank)"), +    ("article-journal", "Journal Article"), +    ("paper-conference", "Conference Proceeding"), +    ("article", "Article (non-journal)"), +    ("book", "Book"), +    ("chapter", "Book Chapter"), +    ("dataset", "Dataset"), +    ("stub", "Invalid/Stub"),  ]  release_stage_options = [ -    ('', 'Unknown (blank)'), -    ('draft', 'Draft'), -    ('submitted', 'Submitted'), -    ('accepted', 'Accepted'), -    ('published', 'Published'), -    ('updated', 'Updated'), +    ("", "Unknown (blank)"), +    ("draft", "Draft"), +    ("submitted", "Submitted"), +    ("accepted", "Accepted"), +    ("published", "Published"), +    ("updated", "Updated"),  ]  withdrawn_status_options = [ -    ('', 'Not Withdrawn (blank)'), -    ('retracted', 'Retracted'), -    ('withdrawn', 'Withdrawn'), -    ('concern', 'Concern Noted'), -    ('spam', 'Spam'), -    ('legal', 'Legal Taketown'), -    ('safety', 'Public Safety'), -    ('national-security', 'National Security'), +    ("", "Not Withdrawn (blank)"), +    ("retracted", "Retracted"), +    ("withdrawn", "Withdrawn"), +    ("concern", "Concern Noted"), +    ("spam", "Spam"), +    ("legal", "Legal Taketown"), +    ("safety", "Public Safety"), +    ("national-security", "National Security"),  ]  role_type_options = [ -    ('author', 'Author'), -    ('editor', 'Editor'), -    ('translator', 'Translator'), +    ("author", "Author"), +    ("editor", "Editor"), +    ("translator", "Translator"),  ] +  class EntityEditForm(FlaskForm): -    editgroup_id = StringField('Editgroup ID', -        [validators.Optional(True), -         validators.Length(min=26, max=26)]) -    editgroup_description = StringField('Editgroup Description', -        [validators.Optional(True)]) -    edit_description = StringField('Description of Changes', -        [validators.Optional(True)]) +    editgroup_id = StringField( +        "Editgroup ID", [validators.Optional(True), validators.Length(min=26, max=26)] +    ) +    editgroup_description = StringField("Editgroup Description", [validators.Optional(True)]) +    edit_description = StringField("Description of Changes", [validators.Optional(True)]) +  class ReleaseContribForm(FlaskForm):      class Meta:          # this is a sub-form, so disable CSRF          csrf = False -    #surname -    #given_name -    #creator_id (?) -    #orcid (for match?) -    prev_index = HiddenField('prev_revision index', default=None) -    raw_name = StringField('Display Name', -        [validators.DataRequired()]) -    role = SelectField( -        [validators.DataRequired()], -        choices=role_type_options, -        default='author') +    # surname +    # given_name +    # creator_id (?) +    # orcid (for match?) +    prev_index = HiddenField("prev_revision index", default=None) +    raw_name = StringField("Display Name", [validators.DataRequired()]) +    role = SelectField([validators.DataRequired()], choices=role_type_options, default="author") + + +RELEASE_SIMPLE_ATTRS = [ +    "title", +    "original_title", +    "work_id", +    "container_id", +    "release_type", +    "release_stage", +    "withdrawn_status", +    "release_date", +    "release_year", +    "volume", +    "issue", +    "pages", +    "publisher", +    "language", +    "license_slug", +] -RELEASE_SIMPLE_ATTRS = ['title', 'original_title', 'work_id', 'container_id', -    'release_type', 'release_stage', 'withdrawn_status', 'release_date', -    'release_year', 'volume', 'issue', 'pages', 'publisher', 'language', -    'license_slug'] +RELEASE_EXTID_ATTRS = ["doi", "wikidata_qid", "isbn13", "pmid", "pmcid"] -RELEASE_EXTID_ATTRS = ['doi', 'wikidata_qid', 'isbn13', 'pmid', 'pmcid']  def valid_year(form, field):      if field.data > datetime.date.today().year + 5: -        raise ValidationError( -            f"Year is too far in the future: {field.data}") +        raise ValidationError(f"Year is too far in the future: {field.data}")      if field.data < 10: -        raise ValidationError( -            f"Year is too far in the past: {field.data}") +        raise ValidationError(f"Year is too far in the past: {field.data}") +  def valid_2char_ascii(form, field): -    if len(field.data) != 2 or len(field.data.encode('utf-8')) != 2 or not field.data.isalpha() or field.data != field.data.lower(): -        raise ValidationError( -            f"Must be 2-character ISO format, lower case: {field.data}") +    if ( +        len(field.data) != 2 +        or len(field.data.encode("utf-8")) != 2 +        or not field.data.isalpha() +        or field.data != field.data.lower() +    ): +        raise ValidationError(f"Must be 2-character ISO format, lower case: {field.data}") +  class ReleaseEntityForm(EntityEditForm):      """ @@ -117,50 +131,52 @@ class ReleaseEntityForm(EntityEditForm):      - field types: fatcat id      - date      """ -    title = StringField('Title', -        [validators.DataRequired()]) -    original_title = StringField('Title in Original Language (if different)') -    work_id = StringField('Work FCID', -        [validators.Optional(True), -         validators.Length(min=26, max=26)]) -    container_id = StringField('Container FCID', -        [validators.Optional(True), -         validators.Length(min=26, max=26)]) -    release_type = SelectField('Release Type', -        [validators.DataRequired()], -        choices=release_type_options, -        default='') + +    title = StringField("Title", [validators.DataRequired()]) +    original_title = StringField("Title in Original Language (if different)") +    work_id = StringField( +        "Work FCID", [validators.Optional(True), validators.Length(min=26, max=26)] +    ) +    container_id = StringField( +        "Container FCID", [validators.Optional(True), validators.Length(min=26, max=26)] +    ) +    release_type = SelectField( +        "Release Type", [validators.DataRequired()], choices=release_type_options, default="" +    )      release_stage = SelectField(choices=release_stage_options) -    withdrawn_status = SelectField("Withdrawn Status", +    withdrawn_status = SelectField( +        "Withdrawn Status",          [validators.Optional(True)],          choices=withdrawn_status_options, -        default='') -    release_date = DateField('Release Date', -        [validators.Optional(True)]) -    release_year = IntegerField('Release Year', -        [validators.Optional(True), valid_year]) -    doi = StringField('DOI', -        [validators.Regexp(r'^10\..*\/.*', message="DOI must be valid"), -         validators.Optional(True)]) -    wikidata_qid = StringField('Wikidata QID') -    isbn13 = StringField('ISBN-13') -    pmid = StringField('PubMed Id') -    pmcid = StringField('PubMed Central Id') -    #core_id -    #arxiv_id -    #jstor_id -    #oai -    #hdl -    volume = StringField('Volume') -    issue = StringField('Issue') -    pages = StringField('Pages') -    publisher = StringField('Publisher (optional)') -    language = StringField('Language (code)', -        [validators.Optional(True), valid_2char_ascii]) -    license_slug = StringField('License (slug)') +        default="", +    ) +    release_date = DateField("Release Date", [validators.Optional(True)]) +    release_year = IntegerField("Release Year", [validators.Optional(True), valid_year]) +    doi = StringField( +        "DOI", +        [ +            validators.Regexp(r"^10\..*\/.*", message="DOI must be valid"), +            validators.Optional(True), +        ], +    ) +    wikidata_qid = StringField("Wikidata QID") +    isbn13 = StringField("ISBN-13") +    pmid = StringField("PubMed Id") +    pmcid = StringField("PubMed Central Id") +    # core_id +    # arxiv_id +    # jstor_id +    # oai +    # hdl +    volume = StringField("Volume") +    issue = StringField("Issue") +    pages = StringField("Pages") +    publisher = StringField("Publisher (optional)") +    language = StringField("Language (code)", [validators.Optional(True), valid_2char_ascii]) +    license_slug = StringField("License (slug)")      contribs = FieldList(FormField(ReleaseContribForm)) -    #refs -    #abstracts +    # refs +    # abstracts      @staticmethod      def from_entity(re): @@ -183,7 +199,7 @@ class ReleaseEntityForm(EntityEditForm):          return ref      def to_entity(self): -        assert(self.title.data) +        assert self.title.data          entity = ReleaseEntity(title=self.title.data, ext_ids=ReleaseExtIds())          self.update_entity(entity)          return entity @@ -198,13 +214,13 @@ class ReleaseEntityForm(EntityEditForm):          for simple_attr in RELEASE_SIMPLE_ATTRS:              a = getattr(self, simple_attr).data              # special case blank strings -            if a == '': +            if a == "":                  a = None              setattr(re, simple_attr, a)          for extid_attr in RELEASE_EXTID_ATTRS:              a = getattr(self, extid_attr).data              # special case blank strings -            if a == '': +            if a == "":                  a = None              setattr(re.ext_ids, extid_attr, a)          if self.release_date.data: @@ -219,7 +235,7 @@ class ReleaseEntityForm(EntityEditForm):              old_contribs = []              re.contribs = []          for c in self.contribs: -            if c.prev_index.data not in ('', None): +            if c.prev_index.data not in ("", None):                  rc = old_contribs[int(c.prev_index.data)]                  rc.role = c.role.data or None                  rc.raw_name = c.raw_name.data or None @@ -232,40 +248,52 @@ class ReleaseEntityForm(EntityEditForm):          if self.edit_description.data:              re.edit_extra = dict(description=self.edit_description.data) +  container_type_options = ( -    ('', 'Unknown (blank)'), -    ('journal', 'Scholarly Journal'), -    ('proceedings', 'Proceedings'), -    ('book-series', 'Book Series'), -    ('blog', 'Blog'), -    ('magazine', 'Magazine'), -    ('trade', 'Trade Magazine'), -    ('test', 'Test / Dummy'), +    ("", "Unknown (blank)"), +    ("journal", "Scholarly Journal"), +    ("proceedings", "Proceedings"), +    ("book-series", "Book Series"), +    ("blog", "Blog"), +    ("magazine", "Magazine"), +    ("trade", "Trade Magazine"), +    ("test", "Test / Dummy"),  ) -CONTAINER_SIMPLE_ATTRS = ['name', 'container_type', 'publisher', 'issnl', -    'wikidata_qid', 'issne', 'issnp'] -CONTAINER_EXTRA_ATTRS = ['original_name', 'country'] +CONTAINER_SIMPLE_ATTRS = [ +    "name", +    "container_type", +    "publisher", +    "issnl", +    "wikidata_qid", +    "issne", +    "issnp", +] +CONTAINER_EXTRA_ATTRS = ["original_name", "country"] +  class ContainerEntityForm(EntityEditForm): -    name = StringField('Name/Title', -        [validators.DataRequired()]) -    container_type = SelectField('Container Type', +    name = StringField("Name/Title", [validators.DataRequired()]) +    container_type = SelectField( +        "Container Type",          [validators.Optional(True)],          choices=container_type_options, -        default='') +        default="", +    )      publisher = StringField("Publisher")      issnl = StringField("ISSN-L (linking)")      issne = StringField("ISSN (electronic)")      issnp = StringField("ISSN (print)")      original_name = StringField("Name in Original Language (if different)") -    country = StringField("Country of Publication (ISO code)", -        [validators.Optional(True), valid_2char_ascii]) -    wikidata_qid = StringField('Wikidata QID') +    country = StringField( +        "Country of Publication (ISO code)", [validators.Optional(True), valid_2char_ascii] +    ) +    wikidata_qid = StringField("Wikidata QID")      urls = FieldList( -        StringField("Container URLs", -            [validators.DataRequired(), -             validators.URL(require_tld=False)])) +        StringField( +            "Container URLs", [validators.DataRequired(), validators.URL(require_tld=False)] +        ) +    )      @staticmethod      def from_entity(ce): @@ -281,13 +309,13 @@ class ContainerEntityForm(EntityEditForm):                  if ce.extra.get(k):                      a = getattr(cef, k)                      a.data = ce.extra[k] -            if ce.extra.get('urls'): -                for url in ce.extra['urls']: +            if ce.extra.get("urls"): +                for url in ce.extra["urls"]:                      cef.urls.append_entry(url)          return cef      def to_entity(self): -        assert(self.name.data) +        assert self.name.data          entity = ContainerEntity(name=self.name.data)          self.update_entity(entity)          return entity @@ -302,70 +330,66 @@ class ContainerEntityForm(EntityEditForm):          for simple_attr in CONTAINER_SIMPLE_ATTRS:              a = getattr(self, simple_attr).data              # special case blank strings -            if a == '': +            if a == "":                  a = None              setattr(ce, simple_attr, a)          if not ce.extra:              ce.extra = dict()          for extra_attr in CONTAINER_EXTRA_ATTRS:              a = getattr(self, extra_attr).data -            if a and a != '': +            if a and a != "":                  ce.extra[extra_attr] = a          extra_urls = []          for url in self.urls:              extra_urls.append(url.data)          if extra_urls: -            ce.extra['urls'] = extra_urls +            ce.extra["urls"] = extra_urls          if self.edit_description.data:              ce.edit_extra = dict(description=self.edit_description.data)          if not ce.extra:              ce.extra = None +  url_rel_options = [ -    ('web', 'Public Web'), -    ('webarchive', 'Web Archive'), -    ('repository', 'Repository'), -    ('archive', 'Preservation Archive'), -    ('academicsocial', 'Academic Social Network'), -    ('publisher', 'Publisher'), -    ('dweb', 'Decentralized Web'), -    ('aggregator', 'Aggregator'), +    ("web", "Public Web"), +    ("webarchive", "Web Archive"), +    ("repository", "Repository"), +    ("archive", "Preservation Archive"), +    ("academicsocial", "Academic Social Network"), +    ("publisher", "Publisher"), +    ("dweb", "Decentralized Web"), +    ("aggregator", "Aggregator"),  ] -FILE_SIMPLE_ATTRS = ['size', 'md5', 'sha1', 'sha256', 'mimetype'] +FILE_SIMPLE_ATTRS = ["size", "md5", "sha1", "sha256", "mimetype"] +  class FileUrlForm(FlaskForm):      class Meta:          # this is a sub-form, so disable CSRF          csrf = False -    url = StringField('Display Name', -        [validators.DataRequired(), -         validators.URL(require_tld=False)]) -    rel = SelectField( -        [validators.DataRequired()], -        choices=url_rel_options, -        default='web') +    url = StringField( +        "Display Name", [validators.DataRequired(), validators.URL(require_tld=False)] +    ) +    rel = SelectField([validators.DataRequired()], choices=url_rel_options, default="web") +  class FileEntityForm(EntityEditForm):      # TODO: positive definite -    size = IntegerField('Size (bytes)', -        [validators.DataRequired()]) -    md5 = StringField("MD5", -        [validators.Optional(True), -         validators.Length(min=32, max=32)]) -    sha1 = StringField("SHA-1", -        [validators.DataRequired(), -         validators.Length(min=40, max=40)]) -    sha256 = StringField("SHA-256", -        [validators.Optional(True), -         validators.Length(min=64, max=64)]) +    size = IntegerField("Size (bytes)", [validators.DataRequired()]) +    md5 = StringField("MD5", [validators.Optional(True), validators.Length(min=32, max=32)]) +    sha1 = StringField("SHA-1", [validators.DataRequired(), validators.Length(min=40, max=40)]) +    sha256 = StringField( +        "SHA-256", [validators.Optional(True), validators.Length(min=64, max=64)] +    )      urls = FieldList(FormField(FileUrlForm))      mimetype = StringField("Mimetype")      release_ids = FieldList( -        StringField("Release FCID", -            [validators.DataRequired(), -            validators.Length(min=26, max=26)])) +        StringField( +            "Release FCID", [validators.DataRequired(), validators.Length(min=26, max=26)] +        ) +    )      @staticmethod      def from_entity(fe): @@ -386,7 +410,7 @@ class FileEntityForm(EntityEditForm):          return ref      def to_entity(self): -        assert(self.sha1.data) +        assert self.sha1.data          entity = FileEntity()          self.update_entity(entity)          return entity @@ -400,87 +424,92 @@ class FileEntityForm(EntityEditForm):          for simple_attr in FILE_SIMPLE_ATTRS:              a = getattr(self, simple_attr).data              # be flexible about hash capitalization -            if simple_attr in ('md5', 'sha1', 'sha256'): +            if simple_attr in ("md5", "sha1", "sha256"):                  a = a.lower()              # special case blank strings -            if a == '': +            if a == "":                  a = None              setattr(fe, simple_attr, a)          fe.urls = []          for u in self.urls: -            fe.urls.append(FileUrl( -                rel=u.rel.data or None, -                url=u.url.data or None, -            )) +            fe.urls.append( +                FileUrl( +                    rel=u.rel.data or None, +                    url=u.url.data or None, +                ) +            )          fe.release_ids = []          for ri in self.release_ids:              fe.release_ids.append(ri.data)          if self.edit_description.data:              fe.edit_extra = dict(description=self.edit_description.data) +  INGEST_TYPE_OPTIONS = [ -    ('pdf', 'PDF Fulltext'), -    ('html', 'HTML Fulltext'), -    ('xml', 'XML Fulltext'), +    ("pdf", "PDF Fulltext"), +    ("html", "HTML Fulltext"), +    ("xml", "XML Fulltext"),  ] +  class SavePaperNowForm(FlaskForm): -    base_url = StringField( -        "URL", -        [validators.DataRequired(), -         validators.URL()]) +    base_url = StringField("URL", [validators.DataRequired(), validators.URL()])      ingest_type = SelectField( -        "Content Type", -        [validators.DataRequired()], -        choices=INGEST_TYPE_OPTIONS, -        default='pdf') +        "Content Type", [validators.DataRequired()], choices=INGEST_TYPE_OPTIONS, default="pdf" +    )      release_stage = SelectField(          "Publication Stage",          [validators.DataRequired()],          choices=release_stage_options, -        default='') +        default="", +    ) -    def to_ingest_request(self, release, ingest_request_source='savepapernow'): +    def to_ingest_request(self, release, ingest_request_source="savepapernow"):          base_url = self.base_url.data          ext_ids = release.ext_ids.to_dict()          # by default this dict has a bunch of empty values          ext_ids = dict([(k, v) for (k, v) in ext_ids.items() if v])          ingest_request = { -            'ingest_type': self.ingest_type.data, -            'ingest_request_source': ingest_request_source, -            'link_source': 'spn', -            'link_source_id': release.ident, -            'base_url': base_url, -            'fatcat': { -                'release_ident': release.ident, -                'work_ident': release.work_id, +            "ingest_type": self.ingest_type.data, +            "ingest_request_source": ingest_request_source, +            "link_source": "spn", +            "link_source_id": release.ident, +            "base_url": base_url, +            "fatcat": { +                "release_ident": release.ident, +                "work_ident": release.work_id,              }, -            'ext_ids': ext_ids, +            "ext_ids": ext_ids,          }          if self.release_stage.data: -            ingest_request['release_stage'] = self.release_stage.data +            ingest_request["release_stage"] = self.release_stage.data          if release.ext_ids.doi and base_url == "https://doi.org/{}".format(release.ext_ids.doi): -            ingest_request['link_source'] = 'doi' -            ingest_request['link_source_id'] = release.ext_ids.doi -        elif release.ext_ids.arxiv and base_url == "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv): -            ingest_request['link_source'] = 'arxiv' -            ingest_request['link_source_id'] = release.ext_ids.arxiv +            ingest_request["link_source"] = "doi" +            ingest_request["link_source_id"] = release.ext_ids.doi +        elif release.ext_ids.arxiv and base_url == "https://arxiv.org/pdf/{}.pdf".format( +            release.ext_ids.arxiv +        ): +            ingest_request["link_source"] = "arxiv" +            ingest_request["link_source_id"] = release.ext_ids.arxiv          return ingest_request +  def valid_toml(form, field):      try:          toml.loads(field.data)      except toml.TomlDecodeError as tpe:          raise ValidationError(tpe) +  class EntityTomlForm(EntityEditForm):      toml = TextAreaField(          "TOML", -        [validators.DataRequired(), -         valid_toml, +        [ +            validators.DataRequired(), +            valid_toml,          ],      ) @@ -490,34 +519,42 @@ class EntityTomlForm(EntityEditForm):          Initializes form with TOML version of existing entity          """          etf = EntityTomlForm() -        if entity.state == 'active': -            pop_fields = ['ident', 'state', 'revision', 'redirect'] +        if entity.state == "active": +            pop_fields = ["ident", "state", "revision", "redirect"]          else: -            pop_fields = ['ident', 'state'] +            pop_fields = ["ident", "state"]          # remove "expand" fields -        pop_fields += ['releases', 'container', 'work', 'creators', 'files', 'filesets', 'webcaptures'] +        pop_fields += [ +            "releases", +            "container", +            "work", +            "creators", +            "files", +            "filesets", +            "webcaptures", +        ]          etf.toml.data = entity_to_toml(entity, pop_fields=pop_fields)          return etf -class ReferenceMatchForm(FlaskForm): +class ReferenceMatchForm(FlaskForm):      class Meta:          # this is an API, so disable CSRF          csrf = False -    submit_type = SelectField('submit_type', -        [validators.DataRequired()], -        choices=['parse', 'match']) +    submit_type = SelectField( +        "submit_type", [validators.DataRequired()], choices=["parse", "match"] +    ) -    raw_citation = TextAreaField("Citation String", render_kw={'rows':'3'}) +    raw_citation = TextAreaField("Citation String", render_kw={"rows": "3"})      title = StringField("Title")      journal = StringField("Journal or Conference")      first_author = StringField("First Author") -    #author_names = StringField("Author Names") -    #year = IntegerField('Year Released', +    # author_names = StringField("Author Names") +    # year = IntegerField('Year Released',      #    [validators.Optional(True), valid_year])      year = StringField("Year Released")      date = StringField("Date Released") @@ -539,17 +576,17 @@ class ReferenceMatchForm(FlaskForm):          rmf = ReferenceMatchForm()          rmf.raw_citation.data = raw_citation -        direct_fields = ['title', 'journal', 'volume', 'issue', 'pages'] +        direct_fields = ["title", "journal", "volume", "issue", "pages"]          for k in direct_fields:              if parse_dict.get(k):                  a = getattr(rmf, k)                  a.data = parse_dict[k] -        date = parse_dict.get('date') +        date = parse_dict.get("date")          if date and len(date) >= 4 and date[0:4].isdigit():              rmf.year.data = int(date[0:4]) -        if parse_dict.get('authors'): -            rmf.first_author.data = parse_dict['authors'][0].get('name') +        if parse_dict.get("authors"): +            rmf.first_author.data = parse_dict["authors"][0].get("name")          return rmf diff --git a/python/fatcat_web/graphics.py b/python/fatcat_web/graphics.py index b5a83f6c..c76408cd 100644 --- a/python/fatcat_web/graphics.py +++ b/python/fatcat_web/graphics.py @@ -1,4 +1,3 @@ -  from typing import Dict, List, Tuple  import pygal @@ -15,32 +14,40 @@ def ia_coverage_histogram(rows: List[Tuple]) -> pygal.Graph:      raw_years = [int(r[0]) for r in rows]      years = dict()      if raw_years: -        for y in range(min(raw_years), max(raw_years)+1): +        for y in range(min(raw_years), max(raw_years) + 1):              years[int(y)] = dict(year=int(y), available=0, missing=0)          for r in rows:              if r[1]: -                years[int(r[0])]['available'] = r[2] +                years[int(r[0])]["available"] = r[2]              else: -                years[int(r[0])]['missing'] = r[2] +                years[int(r[0])]["missing"] = r[2] -    years = sorted(years.values(), key=lambda x: x['year']) +    years = sorted(years.values(), key=lambda x: x["year"])      CleanStyle.colors = ("green", "purple")      label_count = len(years)      if len(years) > 20:          label_count = 10 -    chart = pygal.StackedBar(dynamic_print_values=True, style=CleanStyle, -        width=1000, height=500, x_labels_major_count=label_count, -        show_minor_x_labels=False) -    #chart.title = "Perpetual Access Coverage" +    chart = pygal.StackedBar( +        dynamic_print_values=True, +        style=CleanStyle, +        width=1000, +        height=500, +        x_labels_major_count=label_count, +        show_minor_x_labels=False, +    ) +    # chart.title = "Perpetual Access Coverage"      chart.x_title = "Year" -    #chart.y_title = "Releases" -    chart.x_labels = [str(y['year']) for y in years] -    chart.add('via Fatcat', [y['available'] for y in years]) -    chart.add('Missing', [y['missing'] for y in years]) +    # chart.y_title = "Releases" +    chart.x_labels = [str(y["year"]) for y in years] +    chart.add("via Fatcat", [y["available"] for y in years]) +    chart.add("Missing", [y["missing"] for y in years])      return chart -def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: + +def preservation_by_year_histogram( +    rows: List[Dict], merge_shadows: bool = False +) -> pygal.Graph:      """      Note: this returns a raw pygal chart; it does not render it to SVG/PNG @@ -48,7 +55,7 @@ def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False      There is also a 'year' key with float/int value.      """ -    years = sorted(rows, key=lambda x: x['year']) +    years = sorted(rows, key=lambda x: x["year"])      if merge_shadows:          CleanStyle.colors = ("red", "darkolivegreen", "limegreen") @@ -57,23 +64,32 @@ def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False      label_count = len(years)      if len(years) > 30:          label_count = 10 -    chart = pygal.StackedBar(dynamic_print_values=True, style=CleanStyle, -        width=1000, height=500, x_labels_major_count=label_count, -        show_minor_x_labels=False, x_label_rotation=20) -    #chart.title = "Preservation by Year" +    chart = pygal.StackedBar( +        dynamic_print_values=True, +        style=CleanStyle, +        width=1000, +        height=500, +        x_labels_major_count=label_count, +        show_minor_x_labels=False, +        x_label_rotation=20, +    ) +    # chart.title = "Preservation by Year"      chart.x_title = "Year" -    #chart.y_title = "Count" -    chart.x_labels = [str(y['year']) for y in years] +    # chart.y_title = "Count" +    chart.x_labels = [str(y["year"]) for y in years]      if merge_shadows: -        chart.add('None', [y['none'] + y['shadows_only'] for y in years]) +        chart.add("None", [y["none"] + y["shadows_only"] for y in years])      else: -        chart.add('None', [y['none'] for y in years]) -        chart.add('Shadow', [y['shadows_only'] for y in years]) -    chart.add('Dark', [y['dark'] for y in years]) -    chart.add('Bright', [y['bright'] for y in years]) +        chart.add("None", [y["none"] for y in years]) +        chart.add("Shadow", [y["shadows_only"] for y in years]) +    chart.add("Dark", [y["dark"] for y in years]) +    chart.add("Bright", [y["bright"] for y in years])      return chart -def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: + +def preservation_by_date_histogram( +    rows: List[Dict], merge_shadows: bool = False +) -> pygal.Graph:      """      Note: this returns a raw pygal chart; it does not render it to SVG/PNG @@ -81,7 +97,7 @@ def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False      There is also a 'date' key with str value.      """ -    dates = sorted(rows, key=lambda x: x['date']) +    dates = sorted(rows, key=lambda x: x["date"])      if merge_shadows:          CleanStyle.colors = ("red", "darkolivegreen", "limegreen") @@ -90,23 +106,32 @@ def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False      label_count = len(dates)      if len(dates) > 30:          label_count = 10 -    chart = pygal.StackedBar(dynamic_print_values=True, style=CleanStyle, -        width=1000, height=500, x_labels_major_count=label_count, -        show_minor_x_labels=False, x_label_rotation=20) -    #chart.title = "Preservation by Date" +    chart = pygal.StackedBar( +        dynamic_print_values=True, +        style=CleanStyle, +        width=1000, +        height=500, +        x_labels_major_count=label_count, +        show_minor_x_labels=False, +        x_label_rotation=20, +    ) +    # chart.title = "Preservation by Date"      chart.x_title = "Date" -    #chart.y_title = "Count" -    chart.x_labels = [str(y['date']) for y in dates] +    # chart.y_title = "Count" +    chart.x_labels = [str(y["date"]) for y in dates]      if merge_shadows: -        chart.add('None', [y['none'] + y['shadows_only'] for y in dates]) +        chart.add("None", [y["none"] + y["shadows_only"] for y in dates])      else: -        chart.add('None', [y['none'] for y in dates]) -        chart.add('Shadow', [y['shadows_only'] for y in dates]) -    chart.add('Dark', [y['dark'] for y in dates]) -    chart.add('Bright', [y['bright'] for y in dates]) +        chart.add("None", [y["none"] for y in dates]) +        chart.add("Shadow", [y["shadows_only"] for y in dates]) +    chart.add("Dark", [y["dark"] for y in dates]) +    chart.add("Bright", [y["bright"] for y in dates])      return chart -def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph: + +def preservation_by_volume_histogram( +    rows: List[Dict], merge_shadows: bool = False +) -> pygal.Graph:      """      Note: this returns a raw pygal chart; it does not render it to SVG/PNG @@ -114,7 +139,7 @@ def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = Fal      There is also a 'volume' key with str value.      """ -    volumes = sorted(rows, key=lambda x: x['volume']) +    volumes = sorted(rows, key=lambda x: x["volume"])      if merge_shadows:          CleanStyle.colors = ("red", "darkolivegreen", "limegreen") @@ -123,18 +148,24 @@ def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = Fal      label_count = len(volumes)      if len(volumes) >= 30:          label_count = 10 -    chart = pygal.StackedBar(dynamic_print_values=True, style=CleanStyle, -        width=1000, height=500, x_labels_major_count=label_count, -        show_minor_x_labels=False, x_label_rotation=20) -    #chart.title = "Preservation by Volume" +    chart = pygal.StackedBar( +        dynamic_print_values=True, +        style=CleanStyle, +        width=1000, +        height=500, +        x_labels_major_count=label_count, +        show_minor_x_labels=False, +        x_label_rotation=20, +    ) +    # chart.title = "Preservation by Volume"      chart.x_title = "Volume" -    #chart.y_title = "Count" -    chart.x_labels = [str(y['volume']) for y in volumes] +    # chart.y_title = "Count" +    chart.x_labels = [str(y["volume"]) for y in volumes]      if merge_shadows: -        chart.add('None', [y['none'] + y['shadows_only'] for y in volumes]) +        chart.add("None", [y["none"] + y["shadows_only"] for y in volumes])      else: -        chart.add('None', [y['none'] for y in volumes]) -        chart.add('Shadow', [y['shadows_only'] for y in volumes]) -    chart.add('Dark', [y['dark'] for y in volumes]) -    chart.add('Bright', [y['bright'] for y in volumes]) +        chart.add("None", [y["none"] for y in volumes]) +        chart.add("Shadow", [y["shadows_only"] for y in volumes]) +    chart.add("Dark", [y["dark"] for y in volumes]) +    chart.add("Bright", [y["bright"] for y in volumes])      return chart diff --git a/python/fatcat_web/hacks.py b/python/fatcat_web/hacks.py index 9e6f6ab5..06350b41 100644 --- a/python/fatcat_web/hacks.py +++ b/python/fatcat_web/hacks.py @@ -1,17 +1,23 @@ -  import re  STRIP_EXTLINK_XML_RE = re.compile(r"<ext-link.*xlink:type=\"simple\">") +  def strip_extlink_xml(unstr):      unstr = unstr.replace("</ext-link>", "")      unstr = STRIP_EXTLINK_XML_RE.sub("", unstr)      return unstr +  def test_strip_extlink_xml():      assert strip_extlink_xml("asdf") == "asdf" -    assert strip_extlink_xml("""LOCKSS (2014) Available: <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://lockss.org/" xlink:type="simple">http://lockss.org/</ext-link>. Accessed: 2014 November 1.""") == \ -    """LOCKSS (2014) Available: http://lockss.org/. Accessed: 2014 November 1.""" +    assert ( +        strip_extlink_xml( +            """LOCKSS (2014) Available: <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://lockss.org/" xlink:type="simple">http://lockss.org/</ext-link>. Accessed: 2014 November 1.""" +        ) +        == """LOCKSS (2014) Available: http://lockss.org/. Accessed: 2014 November 1.""" +    ) +  def wayback_suffix(entity):      """ diff --git a/python/fatcat_web/kafka.py b/python/fatcat_web/kafka.py index 1d7288af..36dafade 100644 --- a/python/fatcat_web/kafka.py +++ b/python/fatcat_web/kafka.py @@ -1,4 +1,3 @@ -  import requests  from fatcat_web import Config @@ -20,9 +19,9 @@ def kafka_pixy_produce(topic, msg, key=None, sync=True, timeout=25):      params = dict()      if key: -        params['key'] = key +        params["key"] = key      if sync: -        params['sync'] = True +        params["sync"] = True      resp = requests.post(          "{}/topics/{}/messages".format(Config.KAFKA_PIXY_ENDPOINT, topic),          params=params, @@ -31,4 +30,4 @@ def kafka_pixy_produce(topic, msg, key=None, sync=True, timeout=25):          timeout=timeout,      )      resp.raise_for_status() -    #print(resp.json()) +    # print(resp.json()) diff --git a/python/fatcat_web/ref_routes.py b/python/fatcat_web/ref_routes.py index eed3f1df..6a5eb064 100644 --- a/python/fatcat_web/ref_routes.py +++ b/python/fatcat_web/ref_routes.py @@ -28,10 +28,12 @@ from fatcat_web.entity_helpers import generic_get_entity  from fatcat_web.forms import ReferenceMatchForm -def _refs_web(direction, release_ident=None, work_ident=None, openlibrary_id=None, wikipedia_article=None) -> RefHits: -    offset = request.args.get('offset', '0') +def _refs_web( +    direction, release_ident=None, work_ident=None, openlibrary_id=None, wikipedia_article=None +) -> RefHits: +    offset = request.args.get("offset", "0")      offset = max(0, int(offset)) if offset.isnumeric() else 0 -    limit = request.args.get('limit', '30') +    limit = request.args.get("limit", "30")      limit = min(max(0, int(limit)), 100) if limit.isnumeric() else 30      if direction == "in":          hits = get_inbound_refs( @@ -66,144 +68,227 @@ def _refs_web(direction, release_ident=None, work_ident=None, openlibrary_id=Non      return hits -@app.route('/release/<string(length=26):ident>/refs-in', methods=['GET']) +@app.route("/release/<string(length=26):ident>/refs-in", methods=["GET"])  def release_view_refs_inbound(ident):      if request.accept_mimetypes.best == "application/json":          return release_view_refs_inbound_json(ident)      release = generic_get_entity("release", ident)      hits = _refs_web("in", release_ident=ident) -    return render_template('release_view_fuzzy_refs.html', direction="in", entity=release, hits=hits), 200 +    return ( +        render_template( +            "release_view_fuzzy_refs.html", direction="in", entity=release, hits=hits +        ), +        200, +    ) -@app.route('/release/<string(length=26):ident>/refs-out', methods=['GET']) +@app.route("/release/<string(length=26):ident>/refs-out", methods=["GET"])  def release_view_refs_outbound(ident):      if request.accept_mimetypes.best == "application/json":          return release_view_refs_outbound_json(ident)      release = generic_get_entity("release", ident)      hits = _refs_web("out", release_ident=ident) -    return render_template('release_view_fuzzy_refs.html', direction="out", entity=release, hits=hits), 200 +    return ( +        render_template( +            "release_view_fuzzy_refs.html", direction="out", entity=release, hits=hits +        ), +        200, +    ) -@app.route('/openlibrary/OL<int:id_num>W/refs-in', methods=['GET']) + +@app.route("/openlibrary/OL<int:id_num>W/refs-in", methods=["GET"])  def openlibrary_view_refs_inbound(id_num):      if request.accept_mimetypes.best == "application/json":          return openlibrary_view_refs_inbound_json(id_num)      openlibrary_id = f"OL{id_num}W"      hits = _refs_web("in", openlibrary_id=openlibrary_id) -    return render_template('openlibrary_view_fuzzy_refs.html', openlibrary_id=openlibrary_id, direction="in", hits=hits), 200 - -@app.route('/wikipedia/<string(length=2):wiki_lang>:<string:wiki_article>/refs-out', methods=['GET']) +    return ( +        render_template( +            "openlibrary_view_fuzzy_refs.html", +            openlibrary_id=openlibrary_id, +            direction="in", +            hits=hits, +        ), +        200, +    ) + + +@app.route( +    "/wikipedia/<string(length=2):wiki_lang>:<string:wiki_article>/refs-out", methods=["GET"] +)  def wikipedia_view_refs_outbound(wiki_lang: str, wiki_article: str):      if request.accept_mimetypes.best == "application/json":          return wikipedia_view_refs_outbound_json(wiki_lang, wiki_article)      wiki_url = f"https://{wiki_lang}.wikipedia.org/wiki/{wiki_article}" -    wiki_article = wiki_article.replace('_', ' ') +    wiki_article = wiki_article.replace("_", " ")      wikipedia_article = wiki_lang + ":" + wiki_article      hits = _refs_web("out", wikipedia_article=wikipedia_article) -    return render_template('wikipedia_view_fuzzy_refs.html', wiki_article=wiki_article, wiki_lang=wiki_lang, wiki_url=wiki_url, direction="out", hits=hits), 200 - -@app.route('/reference/match', methods=['GET', 'POST']) +    return ( +        render_template( +            "wikipedia_view_fuzzy_refs.html", +            wiki_article=wiki_article, +            wiki_lang=wiki_lang, +            wiki_url=wiki_url, +            direction="out", +            hits=hits, +        ), +        200, +    ) + + +@app.route("/reference/match", methods=["GET", "POST"])  def reference_match():      grobid_status = None      grobid_dict = None      form = ReferenceMatchForm() -    if not form.is_submitted() and request.args.get('submit_type'): +    if not form.is_submitted() and request.args.get("submit_type"):          form = ReferenceMatchForm(request.args) -    if form.is_submitted() or request.args.get('title'): +    if form.is_submitted() or request.args.get("title"):          if form.validate(): -            if form.submit_type.data == 'parse': +            if form.submit_type.data == "parse":                  resp_xml = grobid_api_process_citation(form.raw_citation.data)                  if not resp_xml:                      grobid_status = "failed" -                    return render_template('reference_match.html', form=form, grobid_status=grobid_status), 400 +                    return ( +                        render_template( +                            "reference_match.html", form=form, grobid_status=grobid_status +                        ), +                        400, +                    )                  grobid_dict = transform_grobid_ref_xml(resp_xml)                  if not grobid_dict:                      grobid_status = "empty" -                    return render_template('reference_match.html', form=form, grobid_status=grobid_status), 200 -                #print(grobid_dict) +                    return ( +                        render_template( +                            "reference_match.html", form=form, grobid_status=grobid_status +                        ), +                        200, +                    ) +                # print(grobid_dict)                  release_stub = grobid_ref_to_release(grobid_dict)                  # remove empty values from GROBID parsed dict                  grobid_dict = {k: v for k, v in grobid_dict.items() if v is not None}                  form = ReferenceMatchForm.from_grobid_parse(grobid_dict, form.raw_citation.data)                  grobid_status = "success" -                matches = close_fuzzy_release_matches(es_client=app.es_client, release=release_stub, match_limit=10) or [] -            elif form.submit_type.data == 'match': -                matches = close_fuzzy_biblio_matches(es_client=app.es_client, biblio=form.data, match_limit=10) or [] +                matches = ( +                    close_fuzzy_release_matches( +                        es_client=app.es_client, release=release_stub, match_limit=10 +                    ) +                    or [] +                ) +            elif form.submit_type.data == "match": +                matches = ( +                    close_fuzzy_biblio_matches( +                        es_client=app.es_client, biblio=form.data, match_limit=10 +                    ) +                    or [] +                )              else:                  raise NotImplementedError()              for m in matches:                  # expand releases more completely -                m.release = api.get_release(m.release.ident, expand="container,files,filesets,webcaptures", hide="abstract,refs") +                m.release = api.get_release( +                    m.release.ident, +                    expand="container,files,filesets,webcaptures", +                    hide="abstract,refs", +                )                  # hack in access options                  m.access_options = release_access_options(m.release) -            return render_template('reference_match.html', form=form, grobid_dict=grobid_dict, grobid_status=grobid_status, matches=matches), 200 +            return ( +                render_template( +                    "reference_match.html", +                    form=form, +                    grobid_dict=grobid_dict, +                    grobid_status=grobid_status, +                    matches=matches, +                ), +                200, +            )          elif form.errors: -            return render_template('reference_match.html', form=form), 400 +            return render_template("reference_match.html", form=form), 400 -    return render_template('reference_match.html', form=form), 200 +    return render_template("reference_match.html", form=form), 200  ### Pseudo-APIs ############################################################# -@app.route('/release/<string(length=26):ident>/refs-out.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/release/<string(length=26):ident>/refs-out.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def release_view_refs_outbound_json(ident):      hits = _refs_web("out", release_ident=ident)      return Response(hits.json(exclude_unset=True), mimetype="application/json") -@app.route('/release/<string(length=26):ident>/refs-in.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) +@app.route("/release/<string(length=26):ident>/refs-in.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def release_view_refs_inbound_json(ident):      hits = _refs_web("in", release_ident=ident)      return Response(hits.json(exclude_unset=True), mimetype="application/json") -@app.route('/openlibrary/OL<int:id_num>W/refs-in.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/openlibrary/OL<int:id_num>W/refs-in.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def openlibrary_view_refs_inbound_json(id_num):      openlibrary_id = f"OL{id_num}W"      hits = _refs_web("in", openlibrary_id=openlibrary_id)      return Response(hits.json(exclude_unset=True), mimetype="application/json") -@app.route('/wikipedia/<string(length=2):wiki_lang>:<string:wiki_article>/refs-out.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route( +    "/wikipedia/<string(length=2):wiki_lang>:<string:wiki_article>/refs-out.json", +    methods=["GET", "OPTIONS"], +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def wikipedia_view_refs_outbound_json(wiki_lang: str, wiki_article: str): -    wiki_article = wiki_article.replace('_', ' ') +    wiki_article = wiki_article.replace("_", " ")      wikipedia_article = wiki_lang + ":" + wiki_article      hits = _refs_web("out", wikipedia_article=wikipedia_article)      return Response(hits.json(exclude_unset=True), mimetype="application/json") -@app.route('/reference/match.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) +@app.route("/reference/match.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def reference_match_json():      form = ReferenceMatchForm(request.args)      if form.validate(): -        if form.submit_type.data == 'match': -            matches = close_fuzzy_biblio_matches(es_client=app.es_client, biblio=form.data, match_limit=10) or [] +        if form.submit_type.data == "match": +            matches = ( +                close_fuzzy_biblio_matches( +                    es_client=app.es_client, biblio=form.data, match_limit=10 +                ) +                or [] +            )          else:              raise NotImplementedError()          resp = []          for m in matches:              # expand releases more completely -            m.release = api.get_release(m.release.ident, expand="container,files,filesets,webcaptures", hide="abstract,refs") +            m.release = api.get_release( +                m.release.ident, +                expand="container,files,filesets,webcaptures", +                hide="abstract,refs", +            )              # hack in access options              m.access_options = release_access_options(m.release)              # and manually convert to dict (for jsonify)              info = m.__dict__ -            info['release'] = entity_to_dict(m.release) -            info['access_options'] = [o.dict() for o in m.access_options] +            info["release"] = entity_to_dict(m.release) +            info["access_options"] = [o.dict() for o in m.access_options]              resp.append(info)          return jsonify(resp), 200      else: -        return Response(json.dumps(dict(errors=form.errors)), mimetype="application/json", status=400) +        return Response( +            json.dumps(dict(errors=form.errors)), mimetype="application/json", status=400 +        ) diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index fc94da66..e6963dbc 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -1,4 +1,3 @@ -  import json  import os @@ -72,7 +71,8 @@ from fatcat_web.search import (  ### Generic Entity Views #################################################### -@app.route('/container/<string(length=26):ident>/history', methods=['GET']) + +@app.route("/container/<string(length=26):ident>/history", methods=["GET"])  def container_history(ident):      try:          entity = api.get_container(ident) @@ -80,82 +80,82 @@ def container_history(ident):      except ApiException as ae:          app.log.info(ae)          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="container", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="container", entity=entity, history=history +    ) -@app.route('/creator/<string(length=26):ident>/history', methods=['GET']) + +@app.route("/creator/<string(length=26):ident>/history", methods=["GET"])  def creator_history(ident):      try:          entity = api.get_creator(ident)          history = api.get_creator_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="creator", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="creator", entity=entity, history=history +    ) -@app.route('/file/<string(length=26):ident>/history', methods=['GET']) + +@app.route("/file/<string(length=26):ident>/history", methods=["GET"])  def file_history(ident):      try:          entity = api.get_file(ident)          history = api.get_file_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="file", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="file", entity=entity, history=history +    ) -@app.route('/fileset/<string(length=26):ident>/history', methods=['GET']) + +@app.route("/fileset/<string(length=26):ident>/history", methods=["GET"])  def fileset_history(ident):      try:          entity = api.get_fileset(ident)          history = api.get_fileset_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="fileset", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="fileset", entity=entity, history=history +    ) -@app.route('/webcapture/<string(length=26):ident>/history', methods=['GET']) + +@app.route("/webcapture/<string(length=26):ident>/history", methods=["GET"])  def webcapture_history(ident):      try:          entity = api.get_webcapture(ident)          history = api.get_webcapture_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="webcapture", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="webcapture", entity=entity, history=history +    ) + -@app.route('/release/<string(length=26):ident>/history', methods=['GET']) +@app.route("/release/<string(length=26):ident>/history", methods=["GET"])  def release_history(ident):      try:          entity = api.get_release(ident)          history = api.get_release_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="release", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="release", entity=entity, history=history +    ) + -@app.route('/work/<string(length=26):ident>/history', methods=['GET']) +@app.route("/work/<string(length=26):ident>/history", methods=["GET"])  def work_history(ident):      try:          entity = api.get_work(ident)          history = api.get_work_history(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('entity_history.html', -        entity_type="work", -        entity=entity, -        history=history) +    return render_template( +        "entity_history.html", entity_type="work", entity=entity, history=history +    ) +  def generic_lookup_view(entity_type, lookup_template, extid_types, lookup_lambda):      extid = None @@ -172,81 +172,106 @@ def generic_lookup_view(entity_type, lookup_template, extid_types, lookup_lambda          resp = lookup_lambda({extid: extid_value})      except ValueError:          return make_response( -            render_template(lookup_template, -                lookup_key=extid, -                lookup_value=extid_value, -                lookup_error=400), -            400) +            render_template( +                lookup_template, lookup_key=extid, lookup_value=extid_value, lookup_error=400 +            ), +            400, +        )      except ApiException as ae:          if ae.status == 404 or ae.status == 400:              return make_response( -                render_template(lookup_template, +                render_template( +                    lookup_template,                      lookup_key=extid,                      lookup_value=extid_value, -                    lookup_error=ae.status), -                ae.status) +                    lookup_error=ae.status, +                ), +                ae.status, +            )          else:              app.log.info(ae)              raise ae -    return redirect('/{}/{}'.format(entity_type, resp.ident)) +    return redirect("/{}/{}".format(entity_type, resp.ident)) + -@app.route('/container/lookup', methods=['GET']) +@app.route("/container/lookup", methods=["GET"])  def container_lookup():      return generic_lookup_view( -        'container', -        'container_lookup.html', -        ('issn', 'issne', 'issnp', 'issnl', 'wikidata_qid'), -        lambda p: api.lookup_container(**p)) +        "container", +        "container_lookup.html", +        ("issn", "issne", "issnp", "issnl", "wikidata_qid"), +        lambda p: api.lookup_container(**p), +    ) -@app.route('/creator/lookup', methods=['GET']) + +@app.route("/creator/lookup", methods=["GET"])  def creator_lookup():      return generic_lookup_view( -        'creator', -        'creator_lookup.html', -        ('orcid', 'wikidata_qid'), -        lambda p: api.lookup_creator(**p)) +        "creator", +        "creator_lookup.html", +        ("orcid", "wikidata_qid"), +        lambda p: api.lookup_creator(**p), +    ) -@app.route('/file/lookup', methods=['GET']) + +@app.route("/file/lookup", methods=["GET"])  def file_lookup():      return generic_lookup_view( -        'file', -        'file_lookup.html', -        ('md5', 'sha1', 'sha256'), -        lambda p: api.lookup_file(**p)) +        "file", "file_lookup.html", ("md5", "sha1", "sha256"), lambda p: api.lookup_file(**p) +    ) -@app.route('/fileset/lookup', methods=['GET']) + +@app.route("/fileset/lookup", methods=["GET"])  def fileset_lookup():      abort(404) -@app.route('/webcapture/lookup', methods=['GET']) + +@app.route("/webcapture/lookup", methods=["GET"])  def webcapture_lookup():      abort(404) -@app.route('/release/lookup', methods=['GET']) + +@app.route("/release/lookup", methods=["GET"])  def release_lookup():      return generic_lookup_view( -        'release', -        'release_lookup.html', -        ('doi', 'wikidata_qid', 'pmid', 'pmcid', 'isbn13', 'jstor', 'arxiv', -         'core', 'ark', 'mag', 'oai', 'hdl'), -        lambda p: api.lookup_release(**p)) +        "release", +        "release_lookup.html", +        ( +            "doi", +            "wikidata_qid", +            "pmid", +            "pmcid", +            "isbn13", +            "jstor", +            "arxiv", +            "core", +            "ark", +            "mag", +            "oai", +            "hdl", +        ), +        lambda p: api.lookup_release(**p), +    ) + -@app.route('/work/lookup', methods=['GET']) +@app.route("/work/lookup", methods=["GET"])  def work_lookup():      abort(404) +  ### More Generic Entity Views ############################################### +  def generic_entity_view(entity_type, ident, view_template):      entity = generic_get_entity(entity_type, ident)      if entity.state == "redirect": -        return redirect('/{}/{}'.format(entity_type, entity.redirect)) +        return redirect("/{}/{}".format(entity_type, entity.redirect))      elif entity.state == "deleted": -        return render_template('deleted_entity.html', entity_type=entity_type, entity=entity) +        return render_template("deleted_entity.html", entity_type=entity_type, entity=entity)      metadata = entity.to_dict() -    metadata.pop('extra') +    metadata.pop("extra")      entity._metadata = metadata      if view_template == "container_view.html": @@ -258,16 +283,22 @@ def generic_entity_view(entity_type, ident, view_template):              ReleaseQuery(container_id=ident),          ) -    return render_template(view_template, entity_type=entity_type, entity=entity, editgroup_id=None) +    return render_template( +        view_template, entity_type=entity_type, entity=entity, editgroup_id=None +    ) +  def generic_entity_revision_view(entity_type, revision_id, view_template):      entity = generic_get_entity_revision(entity_type, revision_id)      metadata = entity.to_dict() -    metadata.pop('extra') +    metadata.pop("extra")      entity._metadata = metadata -    return render_template(view_template, entity_type=entity_type, entity=entity, editgroup_id=None) +    return render_template( +        view_template, entity_type=entity_type, entity=entity, editgroup_id=None +    ) +  def generic_editgroup_entity_view(editgroup_id, entity_type, ident, view_template):      try: @@ -278,251 +309,354 @@ def generic_editgroup_entity_view(editgroup_id, entity_type, ident, view_templat      entity, edit = generic_get_editgroup_entity(editgroup, entity_type, ident)      if entity.revision is None or entity.state == "deleted": -        return render_template('deleted_entity.html', entity=entity, -            entity_type=entity_type, editgroup=editgroup) +        return render_template( +            "deleted_entity.html", entity=entity, entity_type=entity_type, editgroup=editgroup +        )      metadata = entity.to_dict() -    metadata.pop('extra') +    metadata.pop("extra")      entity._metadata = metadata -    return render_template(view_template, entity_type=entity_type, entity=entity, editgroup=editgroup) +    return render_template( +        view_template, entity_type=entity_type, entity=entity, editgroup=editgroup +    ) -@app.route('/container/<string(length=26):ident>', methods=['GET']) +@app.route("/container/<string(length=26):ident>", methods=["GET"])  def container_view(ident): -    return generic_entity_view('container', ident, 'container_view.html') +    return generic_entity_view("container", ident, "container_view.html") + -@app.route('/container_<string(length=26):ident>', methods=['GET']) +@app.route("/container_<string(length=26):ident>", methods=["GET"])  def container_underscore_view(ident): -    return redirect('/container/{}'.format(ident)) +    return redirect("/container/{}".format(ident)) -@app.route('/container/<string(length=26):ident>/coverage', methods=['GET']) + +@app.route("/container/<string(length=26):ident>/coverage", methods=["GET"])  def container_view_coverage(ident):      # note: there is a special hack to add entity._type_preservation for this endpoint -    return generic_entity_view('container', ident, 'container_view_coverage.html') +    return generic_entity_view("container", ident, "container_view_coverage.html") + -@app.route('/container/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/container/<string(length=26):ident>/metadata", methods=["GET"])  def container_view_metadata(ident): -    return generic_entity_view('container', ident, 'entity_view_metadata.html') +    return generic_entity_view("container", ident, "entity_view_metadata.html") + -@app.route('/container/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/container/rev/<uuid:revision_id>", methods=["GET"])  def container_revision_view(revision_id): -    return generic_entity_revision_view('container', str(revision_id), 'container_view.html') +    return generic_entity_revision_view("container", str(revision_id), "container_view.html") -@app.route('/container/rev/<uuid:revision_id>/metadata', methods=['GET']) + +@app.route("/container/rev/<uuid:revision_id>/metadata", methods=["GET"])  def container_revision_view_metadata(revision_id): -    return generic_entity_revision_view('container', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view( +        "container", str(revision_id), "entity_view_metadata.html" +    ) -@app.route('/editgroup/<editgroup_id>/container/<string(length=26):ident>', methods=['GET']) + +@app.route("/editgroup/<editgroup_id>/container/<string(length=26):ident>", methods=["GET"])  def container_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'container', ident, 'container_view.html') +    return generic_editgroup_entity_view( +        editgroup_id, "container", ident, "container_view.html" +    ) -@app.route('/editgroup/<editgroup_id>/container/<string(length=26):ident>/metadata', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/container/<string(length=26):ident>/metadata", methods=["GET"] +)  def container_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'container', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "container", ident, "entity_view_metadata.html" +    ) -@app.route('/creator/<string(length=26):ident>', methods=['GET']) +@app.route("/creator/<string(length=26):ident>", methods=["GET"])  def creator_view(ident): -    return generic_entity_view('creator', ident, 'creator_view.html') +    return generic_entity_view("creator", ident, "creator_view.html") -@app.route('/creator_<string(length=26):ident>', methods=['GET']) + +@app.route("/creator_<string(length=26):ident>", methods=["GET"])  def creator_underscore_view(ident): -    return redirect('/creator/{}'.format(ident)) +    return redirect("/creator/{}".format(ident)) + -@app.route('/creator/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/creator/<string(length=26):ident>/metadata", methods=["GET"])  def creator_view_metadata(ident): -    return generic_entity_view('creator', ident, 'entity_view_metadata.html') +    return generic_entity_view("creator", ident, "entity_view_metadata.html") + -@app.route('/creator/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/creator/rev/<uuid:revision_id>", methods=["GET"])  def creator_revision_view(revision_id): -    return generic_entity_revision_view('creator', str(revision_id), 'creator_view.html') +    return generic_entity_revision_view("creator", str(revision_id), "creator_view.html") -@app.route('/creator/rev/<uuid:revision_id>/metadata', methods=['GET']) + +@app.route("/creator/rev/<uuid:revision_id>/metadata", methods=["GET"])  def creator_revision_view_metadata(revision_id): -    return generic_entity_revision_view('creator', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view( +        "creator", str(revision_id), "entity_view_metadata.html" +    ) + -@app.route('/editgroup/<editgroup_id>/creator/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/creator/<string(length=26):ident>", methods=["GET"])  def creator_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'creator', ident, 'creator_view.html') +    return generic_editgroup_entity_view(editgroup_id, "creator", ident, "creator_view.html") -@app.route('/editgroup/<editgroup_id>/creator/<string(length=26):ident>/metadata', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/creator/<string(length=26):ident>/metadata", methods=["GET"] +)  def creator_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'creator', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "creator", ident, "entity_view_metadata.html" +    ) -@app.route('/file/<string(length=26):ident>', methods=['GET']) +@app.route("/file/<string(length=26):ident>", methods=["GET"])  def file_view(ident): -    return generic_entity_view('file', ident, 'file_view.html') +    return generic_entity_view("file", ident, "file_view.html") -@app.route('/file_<string(length=26):ident>', methods=['GET']) + +@app.route("/file_<string(length=26):ident>", methods=["GET"])  def file_underscore_view(ident): -    return redirect('/file/{}'.format(ident)) +    return redirect("/file/{}".format(ident)) + -@app.route('/file/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/file/<string(length=26):ident>/metadata", methods=["GET"])  def file_view_metadata(ident): -    return generic_entity_view('file', ident, 'entity_view_metadata.html') +    return generic_entity_view("file", ident, "entity_view_metadata.html") + -@app.route('/file/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/file/rev/<uuid:revision_id>", methods=["GET"])  def file_revision_view(revision_id): -    return generic_entity_revision_view('file', str(revision_id), 'file_view.html') +    return generic_entity_revision_view("file", str(revision_id), "file_view.html") -@app.route('/file/rev/<uuid:revision_id>/metadata', methods=['GET']) + +@app.route("/file/rev/<uuid:revision_id>/metadata", methods=["GET"])  def file_revision_view_metadata(revision_id): -    return generic_entity_revision_view('file', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view("file", str(revision_id), "entity_view_metadata.html") + -@app.route('/editgroup/<editgroup_id>/file/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/file/<string(length=26):ident>", methods=["GET"])  def file_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'file', ident, 'file_view.html') +    return generic_editgroup_entity_view(editgroup_id, "file", ident, "file_view.html") + -@app.route('/editgroup/<editgroup_id>/file/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/file/<string(length=26):ident>/metadata", methods=["GET"])  def file_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'file', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "file", ident, "entity_view_metadata.html" +    ) -@app.route('/fileset/<string(length=26):ident>', methods=['GET']) +@app.route("/fileset/<string(length=26):ident>", methods=["GET"])  def fileset_view(ident): -    return generic_entity_view('fileset', ident, 'fileset_view.html') +    return generic_entity_view("fileset", ident, "fileset_view.html") + -@app.route('/fileset_<string(length=26):ident>', methods=['GET']) +@app.route("/fileset_<string(length=26):ident>", methods=["GET"])  def fileset_underscore_view(ident): -    return redirect('/fileset/{}'.format(ident)) +    return redirect("/fileset/{}".format(ident)) + -@app.route('/fileset/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/fileset/<string(length=26):ident>/metadata", methods=["GET"])  def fileset_view_metadata(ident): -    return generic_entity_view('fileset', ident, 'entity_view_metadata.html') +    return generic_entity_view("fileset", ident, "entity_view_metadata.html") -@app.route('/fileset/rev/<uuid:revision_id>', methods=['GET']) + +@app.route("/fileset/rev/<uuid:revision_id>", methods=["GET"])  def fileset_revision_view(revision_id): -    return generic_entity_revision_view('fileset', str(revision_id), 'fileset_view.html') +    return generic_entity_revision_view("fileset", str(revision_id), "fileset_view.html") + -@app.route('/fileset/rev/<uuid:revision_id>/metadata', methods=['GET']) +@app.route("/fileset/rev/<uuid:revision_id>/metadata", methods=["GET"])  def fileset_revision_view_metadata(revision_id): -    return generic_entity_revision_view('fileset', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view( +        "fileset", str(revision_id), "entity_view_metadata.html" +    ) + -@app.route('/editgroup/<editgroup_id>/fileset/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/fileset/<string(length=26):ident>", methods=["GET"])  def fileset_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'fileset', ident, 'fileset_view.html') +    return generic_editgroup_entity_view(editgroup_id, "fileset", ident, "fileset_view.html") -@app.route('/editgroup/<editgroup_id>/fileset/<string(length=26):ident>/metadata', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/fileset/<string(length=26):ident>/metadata", methods=["GET"] +)  def fileset_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'fileset', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "fileset", ident, "entity_view_metadata.html" +    ) -@app.route('/webcapture/<string(length=26):ident>', methods=['GET']) +@app.route("/webcapture/<string(length=26):ident>", methods=["GET"])  def webcapture_view(ident): -    return generic_entity_view('webcapture', ident, 'webcapture_view.html') +    return generic_entity_view("webcapture", ident, "webcapture_view.html") -@app.route('/webcapture_<string(length=26):ident>', methods=['GET']) + +@app.route("/webcapture_<string(length=26):ident>", methods=["GET"])  def webcapture_underscore_view(ident): -    return redirect('/webcapture/{}'.format(ident)) +    return redirect("/webcapture/{}".format(ident)) + -@app.route('/webcapture/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/webcapture/<string(length=26):ident>/metadata", methods=["GET"])  def webcapture_view_metadata(ident): -    return generic_entity_view('webcapture', ident, 'entity_view_metadata.html') +    return generic_entity_view("webcapture", ident, "entity_view_metadata.html") + -@app.route('/webcapture/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/webcapture/rev/<uuid:revision_id>", methods=["GET"])  def webcapture_revision_view(revision_id): -    return generic_entity_revision_view('webcapture', str(revision_id), 'webcapture_view.html') +    return generic_entity_revision_view("webcapture", str(revision_id), "webcapture_view.html") -@app.route('/webcapture/rev/<uuid:revision_id>/metadata', methods=['GET']) + +@app.route("/webcapture/rev/<uuid:revision_id>/metadata", methods=["GET"])  def webcapture_revision_view_metadata(revision_id): -    return generic_entity_revision_view('webcapture', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view( +        "webcapture", str(revision_id), "entity_view_metadata.html" +    ) + -@app.route('/editgroup/<editgroup_id>/webcapture/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/webcapture/<string(length=26):ident>", methods=["GET"])  def webcapture_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'webcapture', ident, 'webcapture_view.html') +    return generic_editgroup_entity_view( +        editgroup_id, "webcapture", ident, "webcapture_view.html" +    ) + -@app.route('/editgroup/<editgroup_id>/webcapture/<string(length=26):ident>/metadata', methods=['GET']) +@app.route( +    "/editgroup/<editgroup_id>/webcapture/<string(length=26):ident>/metadata", methods=["GET"] +)  def webcapture_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'webcapture', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "webcapture", ident, "entity_view_metadata.html" +    ) -@app.route('/release/<string(length=26):ident>', methods=['GET']) +@app.route("/release/<string(length=26):ident>", methods=["GET"])  def release_view(ident): -    return generic_entity_view('release', ident, 'release_view.html') +    return generic_entity_view("release", ident, "release_view.html") + -@app.route('/release_<string(length=26):ident>', methods=['GET']) +@app.route("/release_<string(length=26):ident>", methods=["GET"])  def release_underscore_view(ident): -    return redirect('/release/{}'.format(ident)) +    return redirect("/release/{}".format(ident)) + -@app.route('/release/<string(length=26):ident>/contribs', methods=['GET']) +@app.route("/release/<string(length=26):ident>/contribs", methods=["GET"])  def release_view_contribs(ident): -    return generic_entity_view('release', ident, 'release_view_contribs.html') +    return generic_entity_view("release", ident, "release_view_contribs.html") -@app.route('/release/<string(length=26):ident>/references', methods=['GET']) + +@app.route("/release/<string(length=26):ident>/references", methods=["GET"])  def release_view_references(ident): -    return generic_entity_view('release', ident, 'release_view_references.html') +    return generic_entity_view("release", ident, "release_view_references.html") + -@app.route('/release/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/release/<string(length=26):ident>/metadata", methods=["GET"])  def release_view_metadata(ident): -    return generic_entity_view('release', ident, 'entity_view_metadata.html') +    return generic_entity_view("release", ident, "entity_view_metadata.html") + -@app.route('/release/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/release/rev/<uuid:revision_id>", methods=["GET"])  def release_revision_view(revision_id): -    return generic_entity_revision_view('release', str(revision_id), 'release_view.html') +    return generic_entity_revision_view("release", str(revision_id), "release_view.html") -@app.route('/release/rev/<uuid:revision_id>/contribs', methods=['GET']) + +@app.route("/release/rev/<uuid:revision_id>/contribs", methods=["GET"])  def release_revision_view_contribs(revision_id): -    return generic_entity_revision_view('release', str(revision_id), 'release_view_contribs.html') +    return generic_entity_revision_view( +        "release", str(revision_id), "release_view_contribs.html" +    ) + -@app.route('/release/rev/<uuid:revision_id>/references', methods=['GET']) +@app.route("/release/rev/<uuid:revision_id>/references", methods=["GET"])  def release_revision_view_references(revision_id): -    return generic_entity_revision_view('release', str(revision_id), 'release_view_references.html') +    return generic_entity_revision_view( +        "release", str(revision_id), "release_view_references.html" +    ) + -@app.route('/release/rev/<uuid:revision_id>/metadata', methods=['GET']) +@app.route("/release/rev/<uuid:revision_id>/metadata", methods=["GET"])  def release_revision_view_metadata(revision_id): -    return generic_entity_revision_view('release', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view( +        "release", str(revision_id), "entity_view_metadata.html" +    ) + -@app.route('/editgroup/<editgroup_id>/release/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/release/<string(length=26):ident>", methods=["GET"])  def release_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'release', ident, 'release_view.html') +    return generic_editgroup_entity_view(editgroup_id, "release", ident, "release_view.html") -@app.route('/editgroup/<editgroup_id>/release/<string(length=26):ident>/contribs', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/release/<string(length=26):ident>/contribs", methods=["GET"] +)  def release_editgroup_view_contribs(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'release', ident, 'release_view_contribs.html') +    return generic_editgroup_entity_view( +        editgroup_id, "release", ident, "release_view_contribs.html" +    ) -@app.route('/editgroup/<editgroup_id>/release/<string(length=26):ident>/references', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/release/<string(length=26):ident>/references", methods=["GET"] +)  def release_editgroup_view_references(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'release', ident, 'release_view_references.html') +    return generic_editgroup_entity_view( +        editgroup_id, "release", ident, "release_view_references.html" +    ) -@app.route('/editgroup/<editgroup_id>/release/<string(length=26):ident>/metadata', methods=['GET']) + +@app.route( +    "/editgroup/<editgroup_id>/release/<string(length=26):ident>/metadata", methods=["GET"] +)  def release_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'release', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "release", ident, "entity_view_metadata.html" +    ) -@app.route('/work/<string(length=26):ident>', methods=['GET']) +@app.route("/work/<string(length=26):ident>", methods=["GET"])  def work_view(ident): -    return generic_entity_view('work', ident, 'work_view.html') +    return generic_entity_view("work", ident, "work_view.html") -@app.route('/work_<string(length=26):ident>', methods=['GET']) + +@app.route("/work_<string(length=26):ident>", methods=["GET"])  def work_underscore_view(ident): -    return redirect('/work/{}'.format(ident)) +    return redirect("/work/{}".format(ident)) + -@app.route('/work/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/work/<string(length=26):ident>/metadata", methods=["GET"])  def work_view_metadata(ident): -    return generic_entity_view('work', ident, 'entity_view_metadata.html') +    return generic_entity_view("work", ident, "entity_view_metadata.html") + -@app.route('/work/rev/<uuid:revision_id>', methods=['GET']) +@app.route("/work/rev/<uuid:revision_id>", methods=["GET"])  def work_revision_view(revision_id): -    return generic_entity_revision_view('work', str(revision_id), 'work_view.html') +    return generic_entity_revision_view("work", str(revision_id), "work_view.html") -@app.route('/work/rev/<uuid:revision_id>/metadata', methods=['GET']) + +@app.route("/work/rev/<uuid:revision_id>/metadata", methods=["GET"])  def work_revision_view_metadata(revision_id): -    return generic_entity_revision_view('work', str(revision_id), 'entity_view_metadata.html') +    return generic_entity_revision_view("work", str(revision_id), "entity_view_metadata.html") + -@app.route('/editgroup/<editgroup_id>/work/<string(length=26):ident>', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/work/<string(length=26):ident>", methods=["GET"])  def work_editgroup_view(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'work', ident, 'work_view.html') +    return generic_editgroup_entity_view(editgroup_id, "work", ident, "work_view.html") + -@app.route('/editgroup/<editgroup_id>/work/<string(length=26):ident>/metadata', methods=['GET']) +@app.route("/editgroup/<editgroup_id>/work/<string(length=26):ident>/metadata", methods=["GET"])  def work_editgroup_view_metadata(editgroup_id, ident): -    return generic_editgroup_entity_view(editgroup_id, 'work', ident, 'entity_view_metadata.html') +    return generic_editgroup_entity_view( +        editgroup_id, "work", ident, "entity_view_metadata.html" +    )  ### Views ################################################################### -@app.route('/editgroup/<string(length=26):ident>', methods=['GET']) + +@app.route("/editgroup/<string(length=26):ident>", methods=["GET"])  def editgroup_view(ident):      try:          eg = api.get_editgroup(str(ident)) @@ -537,28 +671,28 @@ def editgroup_view(ident):          edit=False,          annotate=False,      ) -    if session.get('editor'): -        user = load_user(session['editor']['editor_id']) -        auth_to['annotate'] = True +    if session.get("editor"): +        user = load_user(session["editor"]["editor_id"]) +        auth_to["annotate"] = True          if user.is_admin or user.editor_id == eg.editor_id: -            auth_to['submit'] = True -            auth_to['edit'] = True +            auth_to["submit"] = True +            auth_to["edit"] = True          if user.is_admin: -            auth_to['accept'] = True -    return render_template('editgroup_view.html', editgroup=eg, -        auth_to=auth_to) +            auth_to["accept"] = True +    return render_template("editgroup_view.html", editgroup=eg, auth_to=auth_to) + -@app.route('/editgroup/<string(length=26):ident>/annotation', methods=['POST']) +@app.route("/editgroup/<string(length=26):ident>/annotation", methods=["POST"])  @login_required  def editgroup_create_annotation(ident):      if not app.testing:          app.csrf.protect() -    comment_markdown = request.form.get('comment_markdown') +    comment_markdown = request.form.get("comment_markdown")      if not comment_markdown:          app.log.info("empty comment field")          abort(400)      # on behalf of user... -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      try:          eg = user_api.get_editgroup(str(ident))          if eg.changelog_index: @@ -571,15 +705,16 @@ def editgroup_create_annotation(ident):      except ApiException as ae:          app.log.info(ae)          raise ae -    return redirect('/editgroup/{}'.format(ident)) +    return redirect("/editgroup/{}".format(ident)) -@app.route('/editgroup/<string(length=26):ident>/accept', methods=['POST']) + +@app.route("/editgroup/<string(length=26):ident>/accept", methods=["POST"])  @login_required  def editgroup_accept(ident):      if not app.testing:          app.csrf.protect()      # on behalf of user... -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      try:          eg = user_api.get_editgroup(str(ident))          if eg.changelog_index: @@ -588,15 +723,16 @@ def editgroup_accept(ident):      except ApiException as ae:          app.log.info(ae)          abort(ae.status) -    return redirect('/editgroup/{}'.format(ident)) +    return redirect("/editgroup/{}".format(ident)) + -@app.route('/editgroup/<string(length=26):ident>/unsubmit', methods=['POST']) +@app.route("/editgroup/<string(length=26):ident>/unsubmit", methods=["POST"])  @login_required  def editgroup_unsubmit(ident):      if not app.testing:          app.csrf.protect()      # on behalf of user... -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      try:          eg = user_api.get_editgroup(str(ident))          if eg.changelog_index: @@ -605,15 +741,16 @@ def editgroup_unsubmit(ident):      except ApiException as ae:          app.log.info(ae)          abort(ae.status) -    return redirect('/editgroup/{}'.format(ident)) +    return redirect("/editgroup/{}".format(ident)) + -@app.route('/editgroup/<string(length=26):ident>/submit', methods=['POST']) +@app.route("/editgroup/<string(length=26):ident>/submit", methods=["POST"])  @login_required  def editgroup_submit(ident):      if not app.testing:          app.csrf.protect()      # on behalf of user... -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      try:          eg = user_api.get_editgroup(str(ident))          if eg.changelog_index: @@ -622,17 +759,19 @@ def editgroup_submit(ident):      except ApiException as ae:          app.log.info(ae)          abort(ae.status) -    return redirect('/editgroup/{}'.format(ident)) +    return redirect("/editgroup/{}".format(ident)) -@app.route('/editor/<string(length=26):ident>', methods=['GET']) + +@app.route("/editor/<string(length=26):ident>", methods=["GET"])  def editor_view(ident):      try:          entity = api.get_editor(ident)      except ApiException as ae:          abort(ae.status) -    return render_template('editor_view.html', editor=entity) +    return render_template("editor_view.html", editor=entity) + -@app.route('/editor/<string(length=26):ident>/editgroups', methods=['GET']) +@app.route("/editor/<string(length=26):ident>/editgroups", methods=["GET"])  def editor_editgroups(ident):      try:          editor = api.get_editor(ident) @@ -642,57 +781,62 @@ def editor_editgroups(ident):              eg.editor = editor      except ApiException as ae:          abort(ae.status) -    return render_template('editor_editgroups.html', editor=editor, -        editgroups=editgroups) +    return render_template("editor_editgroups.html", editor=editor, editgroups=editgroups) -@app.route('/editor/<string(length=26):ident>/annotations', methods=['GET']) + +@app.route("/editor/<string(length=26):ident>/annotations", methods=["GET"])  def editor_annotations(ident):      try:          editor = api.get_editor(ident)          annotations = api.get_editor_annotations(ident, limit=50)      except ApiException as ae:          abort(ae.status) -    return render_template('editor_annotations.html', editor=editor, -        annotations=annotations) +    return render_template("editor_annotations.html", editor=editor, annotations=annotations) + -@app.route('/u/<string:username>', methods=['GET', 'HEAD']) +@app.route("/u/<string:username>", methods=["GET", "HEAD"])  def editor_username_redirect(username):      try:          editor = api.lookup_editor(username=username)      except ApiException as ae:          abort(ae.status) -    return redirect(f'/editor/{editor.editor_id}') +    return redirect(f"/editor/{editor.editor_id}") -@app.route('/changelog', methods=['GET']) + +@app.route("/changelog", methods=["GET"])  def changelog_view():      try: -        #limit = int(request.args.get('limit', 10)) -        entries = api.get_changelog() # TODO: expand="editors" +        # limit = int(request.args.get('limit', 10)) +        entries = api.get_changelog()  # TODO: expand="editors"      except ApiException as ae:          abort(ae.status) -    return render_template('changelog.html', entries=entries) +    return render_template("changelog.html", entries=entries) + -@app.route('/changelog/<int:index>', methods=['GET']) +@app.route("/changelog/<int:index>", methods=["GET"])  def changelog_entry_view(index):      try:          entry = api.get_changelog_entry(int(index))          entry.editgroup.editor = api.get_editor(entry.editgroup.editor_id) -        entry.editgroup.annotations = \ -            api.get_editgroup_annotations(entry.editgroup_id, expand="editors") +        entry.editgroup.annotations = api.get_editgroup_annotations( +            entry.editgroup_id, expand="editors" +        )      except ApiException as ae:          abort(ae.status) -    return render_template('changelog_view.html', entry=entry, editgroup=entry.editgroup) +    return render_template("changelog_view.html", entry=entry, editgroup=entry.editgroup) -@app.route('/reviewable', methods=['GET']) + +@app.route("/reviewable", methods=["GET"])  def reviewable_view():      try: -        #limit = int(request.args.get('limit', 10)) +        # limit = int(request.args.get('limit', 10))          entries = api.get_editgroups_reviewable(expand="editors")      except ApiException as ae:          abort(ae.status) -    return render_template('editgroup_reviewable.html', entries=entries) +    return render_template("editgroup_reviewable.html", entries=entries) + -@app.route('/release/<string(length=26):ident>/save', methods=['GET', 'POST']) +@app.route("/release/<string(length=26):ident>/save", methods=["GET", "POST"])  def release_save(ident):      form = SavePaperNowForm() @@ -704,7 +848,12 @@ def release_save(ident):          abort(ae.status)      if not Config.KAFKA_PIXY_ENDPOINT: -        return render_template('release_save.html', entity=release, form=form, spn_status='not-configured'), 501 +        return ( +            render_template( +                "release_save.html", entity=release, form=form, spn_status="not-configured" +            ), +            501, +        )      if form.is_submitted():          if form.validate_on_submit(): @@ -716,10 +865,20 @@ def release_save(ident):                      json.dumps(msg, sort_keys=True),                  )              except Exception: -                return render_template('release_save.html', entity=release, form=form, spn_status='kafka-error'), 500 -            return render_template('release_save.html', entity=release, form=form, spn_status='success'), 200 +                return ( +                    render_template( +                        "release_save.html", entity=release, form=form, spn_status="kafka-error" +                    ), +                    500, +                ) +            return ( +                render_template( +                    "release_save.html", entity=release, form=form, spn_status="success" +                ), +                200, +            )          elif form.errors: -            return render_template('release_save.html', entity=release, form=form), 400 +            return render_template("release_save.html", entity=release, form=form), 400      # form was not submitted; populate defaults      if release.release_stage: @@ -729,50 +888,57 @@ def release_save(ident):      elif release.ext_ids.arxiv:          form.base_url.data = "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv)      elif release.ext_ids.pmcid: -        form.base_url.data = "https://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.ext_ids.pmcid) +        form.base_url.data = ( +            "https://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format( +                release.ext_ids.pmcid +            ) +        )      elif release.ext_ids.hdl:          form.base_url.data = "https://hdl.handle.net/{}".format(release.ext_ids.hdl) -    return render_template('release_save.html', entity=release, form=form), 200 +    return render_template("release_save.html", entity=release, form=form), 200 +  ### Search ################################################################## -@app.route('/search', methods=['GET', 'POST']) + +@app.route("/search", methods=["GET", "POST"])  def generic_search(): -    if 'q' not in request.args.keys(): -        return redirect('/release/search') -    query = request.args.get('q').strip() +    if "q" not in request.args.keys(): +        return redirect("/release/search") +    query = request.args.get("q").strip()      if len(query.split()) != 1:          # multi-term? must be a real search -        return redirect(url_for('release_search', q=query, generic=1)) +        return redirect(url_for("release_search", q=query, generic=1))      if clean_doi(query): -        return redirect(url_for('release_lookup', doi=clean_doi(query))) +        return redirect(url_for("release_lookup", doi=clean_doi(query)))      if clean_pmcid(query): -        return redirect(url_for('release_lookup', pmcid=clean_pmcid(query))) +        return redirect(url_for("release_lookup", pmcid=clean_pmcid(query)))      if clean_sha1(query): -        return redirect(url_for('file_lookup', sha1=clean_sha1(query))) +        return redirect(url_for("file_lookup", sha1=clean_sha1(query)))      if clean_sha256(query): -        return redirect(url_for('file_lookup', sha256=clean_sha256(query))) +        return redirect(url_for("file_lookup", sha256=clean_sha256(query)))      if clean_issn(query): -        return redirect(url_for('container_lookup', issnl=clean_issn(query))) +        return redirect(url_for("container_lookup", issnl=clean_issn(query)))      if clean_isbn13(query): -        return redirect(url_for('release_lookup', isbn13=clean_isbn13(query))) +        return redirect(url_for("release_lookup", isbn13=clean_isbn13(query)))      if clean_arxiv_id(query): -        return redirect(url_for('release_lookup', arxiv=clean_arxiv_id(query))) +        return redirect(url_for("release_lookup", arxiv=clean_arxiv_id(query)))      if clean_orcid(query): -        return redirect(url_for('creator_lookup', orcid=clean_orcid(query))) +        return redirect(url_for("creator_lookup", orcid=clean_orcid(query))) + +    return redirect(url_for("release_search", q=query, generic=1)) -    return redirect(url_for('release_search', q=query, generic=1)) -@app.route('/release/search', methods=['GET', 'POST']) +@app.route("/release/search", methods=["GET", "POST"])  def release_search(): -    if 'q' not in request.args.keys(): -        return render_template('release_search.html', query=ReleaseQuery(), found=None) +    if "q" not in request.args.keys(): +        return render_template("release_search.html", query=ReleaseQuery(), found=None)      container_found = None -    if request.args.get('generic'): +    if request.args.get("generic"):          container_query = GenericQuery.from_args(request.args)          container_query.limit = 1          try: @@ -784,28 +950,38 @@ def release_search():      try:          found = do_release_search(query)      except FatcatSearchError as fse: -        return render_template('release_search.html', query=query, es_error=fse), fse.status_code -    return render_template('release_search.html', query=query, found=found, container_found=container_found) +        return ( +            render_template("release_search.html", query=query, es_error=fse), +            fse.status_code, +        ) +    return render_template( +        "release_search.html", query=query, found=found, container_found=container_found +    ) + -@app.route('/container/search', methods=['GET', 'POST']) +@app.route("/container/search", methods=["GET", "POST"])  def container_search(): -    if 'q' not in request.args.keys(): -        return render_template('container_search.html', query=GenericQuery(), found=None) +    if "q" not in request.args.keys(): +        return render_template("container_search.html", query=GenericQuery(), found=None)      query = GenericQuery.from_args(request.args)      try:          found = do_container_search(query)      except FatcatSearchError as fse: -        return render_template('container_search.html', query=query, es_error=fse), fse.status_code -    return render_template('container_search.html', query=query, found=found) +        return ( +            render_template("container_search.html", query=query, es_error=fse), +            fse.status_code, +        ) +    return render_template("container_search.html", query=query, found=found) + -@app.route('/coverage/search', methods=['GET', 'POST']) +@app.route("/coverage/search", methods=["GET", "POST"])  def coverage_search(): -    if 'q' not in request.args.keys(): +    if "q" not in request.args.keys():          return render_template( -            'coverage_search.html', +            "coverage_search.html",              query=ReleaseQuery(),              coverage_stats=None,              coverage_type_preservation=None, @@ -817,19 +993,22 @@ def coverage_search():      try:          coverage_stats = get_elastic_search_coverage(query)      except FatcatSearchError as fse: -        return render_template( -            'coverage_search.html', -            query=query, -            coverage_stats=None, -            coverage_type_preservation=None, -            year_histogram_svg=None, -            date_histogram_svg=None, -            es_error=fse, -        ), fse.status_code +        return ( +            render_template( +                "coverage_search.html", +                query=query, +                coverage_stats=None, +                coverage_type_preservation=None, +                year_histogram_svg=None, +                date_histogram_svg=None, +                es_error=fse, +            ), +            fse.status_code, +        )      year_histogram_svg = None      date_histogram_svg = None      coverage_type_preservation = None -    if coverage_stats['total'] > 1: +    if coverage_stats["total"] > 1:          coverage_type_preservation = get_elastic_preservation_by_type(query)          if query.recent:              date_histogram = get_elastic_preservation_by_date(query) @@ -844,7 +1023,7 @@ def coverage_search():                  merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,              ).render_data_uri()      return render_template( -        'coverage_search.html', +        "coverage_search.html",          query=query,          coverage_stats=coverage_stats,          coverage_type_preservation=coverage_type_preservation, @@ -852,16 +1031,20 @@ def coverage_search():          date_histogram_svg=date_histogram_svg,      ) +  def get_changelog_stats():      stats = {}      latest_changelog = api.get_changelog(limit=1)[0] -    stats['changelog'] = {"latest": { -        "index": latest_changelog.index, -        "timestamp": latest_changelog.timestamp.isoformat(), -    }} +    stats["changelog"] = { +        "latest": { +            "index": latest_changelog.index, +            "timestamp": latest_changelog.timestamp.isoformat(), +        } +    }      return stats -@app.route('/stats', methods=['GET']) + +@app.route("/stats", methods=["GET"])  def stats_page():      try:          stats = get_elastic_entity_stats() @@ -869,12 +1052,14 @@ def stats_page():      except Exception as ae:          app.log.error(ae)          abort(503) -    return render_template('stats.html', stats=stats) +    return render_template("stats.html", stats=stats) +  ### Pseudo-APIs ############################################################# -@app.route('/stats.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/stats.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def stats_json():      try:          stats = get_elastic_entity_stats() @@ -884,10 +1069,11 @@ def stats_json():          abort(503)      return jsonify(stats) -@app.route('/container/issnl/<issnl>/stats.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/container/issnl/<issnl>/stats.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_issnl_stats(issnl): -    if not (len(issnl) == 9 and issnl[4] == '-'): +    if not (len(issnl) == 9 and issnl[4] == "-"):          abort(400, "Not a valid ISSN-L: {}".format(issnl))      try:          container = api.lookup_container(issnl=issnl) @@ -900,8 +1086,9 @@ def container_issnl_stats(issnl):          abort(503)      return jsonify(stats) -@app.route('/container/<string(length=26):ident>/stats.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/container/<string(length=26):ident>/stats.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_stats(ident):      try:          container = api.get_container(ident) @@ -914,8 +1101,11 @@ def container_ident_stats(ident):          abort(503)      return jsonify(stats) -@app.route('/container/<string(length=26):ident>/ia_coverage_years.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route( +    "/container/<string(length=26):ident>/ia_coverage_years.json", methods=["GET", "OPTIONS"] +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_ia_coverage_years_json(ident):      try:          container = api.get_container(ident) @@ -927,10 +1117,13 @@ def container_ident_ia_coverage_years_json(ident):          app.log.error(ae)          abort(503)      histogram = [dict(year=h[0], in_ia=h[1], count=h[2]) for h in histogram] -    return jsonify({'container_id': ident, "histogram": histogram}) +    return jsonify({"container_id": ident, "histogram": histogram}) -@app.route('/container/<string(length=26):ident>/ia_coverage_years.svg', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route( +    "/container/<string(length=26):ident>/ia_coverage_years.svg", methods=["GET", "OPTIONS"] +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_ia_coverage_years_svg(ident):      try:          container = api.get_container(ident) @@ -943,8 +1136,11 @@ def container_ident_ia_coverage_years_svg(ident):          abort(503)      return ia_coverage_histogram(histogram).render_response() -@app.route('/container/<string(length=26):ident>/preservation_by_year.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route( +    "/container/<string(length=26):ident>/preservation_by_year.json", methods=["GET", "OPTIONS"] +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_preservation_by_year_json(ident):      try:          container = api.get_container(ident) @@ -956,10 +1152,13 @@ def container_ident_preservation_by_year_json(ident):      except Exception as ae:          app.log.error(ae)          abort(503) -    return jsonify({'container_id': ident, "histogram": histogram}) +    return jsonify({"container_id": ident, "histogram": histogram}) + -@app.route('/container/<string(length=26):ident>/preservation_by_year.svg', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) +@app.route( +    "/container/<string(length=26):ident>/preservation_by_year.svg", methods=["GET", "OPTIONS"] +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_preservation_by_year_svg(ident):      try:          container = api.get_container(ident) @@ -976,8 +1175,12 @@ def container_ident_preservation_by_year_svg(ident):          merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,      ).render_response() -@app.route('/container/<string(length=26):ident>/preservation_by_volume.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route( +    "/container/<string(length=26):ident>/preservation_by_volume.json", +    methods=["GET", "OPTIONS"], +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_preservation_by_volume_json(ident):      try:          container = api.get_container(ident) @@ -988,10 +1191,14 @@ def container_ident_preservation_by_volume_json(ident):      except Exception as ae:          app.log.error(ae)          abort(503) -    return jsonify({'container_id': ident, "histogram": histogram}) +    return jsonify({"container_id": ident, "histogram": histogram}) + -@app.route('/container/<string(length=26):ident>/preservation_by_volume.svg', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) +@app.route( +    "/container/<string(length=26):ident>/preservation_by_volume.svg", +    methods=["GET", "OPTIONS"], +) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def container_ident_preservation_by_volume_svg(ident):      try:          container = api.get_container(ident) @@ -1007,21 +1214,23 @@ def container_ident_preservation_by_volume_svg(ident):          merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,      ).render_response() -@app.route('/release/<string(length=26):ident>.bib', methods=['GET']) + +@app.route("/release/<string(length=26):ident>.bib", methods=["GET"])  def release_bibtex(ident):      try:          entity = api.get_release(ident)      except ApiException as ae:          raise ae      csl = release_to_csl(entity) -    bibtex = citeproc_csl(csl, 'bibtex') +    bibtex = citeproc_csl(csl, "bibtex")      return Response(bibtex, mimetype="text/plain") -@app.route('/release/<string(length=26):ident>/citeproc', methods=['GET']) + +@app.route("/release/<string(length=26):ident>/citeproc", methods=["GET"])  def release_citeproc(ident): -    style = request.args.get('style', 'harvard1') -    is_html = request.args.get('html', False) -    if is_html and is_html.lower() in ('yes', '1', 'true', 'y', 't'): +    style = request.args.get("style", "harvard1") +    is_html = request.args.get("html", False) +    if is_html and is_html.lower() in ("yes", "1", "true", "y", "t"):          is_html = True      else:          is_html = False @@ -1042,67 +1251,75 @@ def release_citeproc(ident):      else:          return Response(cite, mimetype="text/plain") -@app.route('/health.json', methods=['GET', 'OPTIONS']) -@crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type']) + +@app.route("/health.json", methods=["GET", "OPTIONS"]) +@crossdomain(origin="*", headers=["access-control-allow-origin", "Content-Type"])  def health_json(): -    return jsonify({'ok': True}) +    return jsonify({"ok": True})  ### Auth #################################################################### -@app.route('/auth/login') + +@app.route("/auth/login")  def login():      # show the user a list of login options      if not priv_api: -        app.log.warn("This web interface not configured with credentials to actually allow login (other than via token)") -    return render_template('auth_login.html') +        app.log.warn( +            "This web interface not configured with credentials to actually allow login (other than via token)" +        ) +    return render_template("auth_login.html") -@app.route('/auth/ia/login', methods=['GET', 'POST']) + +@app.route("/auth/ia/login", methods=["GET", "POST"])  def ia_xauth_login(): -    if 'email' in request.form: +    if "email" in request.form:          # if a login attempt... -        return handle_ia_xauth(request.form.get('email'), request.form.get('password')) +        return handle_ia_xauth(request.form.get("email"), request.form.get("password"))      # else show form -    return render_template('auth_ia_login.html') +    return render_template("auth_ia_login.html") + -@app.route('/auth/token_login', methods=['GET', 'POST']) +@app.route("/auth/token_login", methods=["GET", "POST"])  def token_login():      # show the user a list of login options -    if 'token' in request.args: -        return handle_token_login(request.args.get('token')) -    if 'token' in request.form: -        return handle_token_login(request.form.get('token')) -    return render_template('auth_token_login.html') +    if "token" in request.args: +        return handle_token_login(request.args.get("token")) +    if "token" in request.form: +        return handle_token_login(request.form.get("token")) +    return render_template("auth_token_login.html") -@app.route('/auth/change_username', methods=['POST']) + +@app.route("/auth/change_username", methods=["POST"])  @login_required  def change_username():      if not app.testing:          app.csrf.protect()      # show the user a list of login options -    if 'username' not in request.form: +    if "username" not in request.form:          abort(400)      # on behalf of user... -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      try: -        editor = user_api.get_editor(session['editor']['editor_id']) -        editor.username = request.form['username'] +        editor = user_api.get_editor(session["editor"]["editor_id"]) +        editor.username = request.form["username"]          editor = user_api.update_editor(editor.editor_id, editor)      except ApiException as ae:          app.log.info(ae)          raise ae      # update our session -    session['editor'] = editor.to_dict() +    session["editor"] = editor.to_dict()      load_user(editor.editor_id) -    return redirect('/auth/account') +    return redirect("/auth/account") + -@app.route('/auth/create_token', methods=['POST']) +@app.route("/auth/create_token", methods=["POST"])  @login_required  def create_auth_token():      if not app.testing:          app.csrf.protect() -    duration_seconds = request.form.get('duration_seconds', None) +    duration_seconds = request.form.get("duration_seconds", None)      if duration_seconds:          try:              duration_seconds = int(duration_seconds) @@ -1114,88 +1331,99 @@ def create_auth_token():      # cookie, so if api_token is valid editor_id is assumed to match. If that      # wasn't true, users could manipulate session cookies and create tokens for      # any user -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      resp = user_api.auth_check() -    assert(resp.success) +    assert resp.success      # generate token using *superuser* privs -    editor_id = session['editor']['editor_id'] +    editor_id = session["editor"]["editor_id"]      try: -        resp = priv_api.create_auth_token(editor_id, -            duration_seconds=duration_seconds) +        resp = priv_api.create_auth_token(editor_id, duration_seconds=duration_seconds)      except ApiException as ae:          app.log.info(ae)          raise ae -    return render_template('auth_token.html', auth_token=resp.token) +    return render_template("auth_token.html", auth_token=resp.token) -@app.route('/auth/logout') + +@app.route("/auth/logout")  def logout():      handle_logout() -    return render_template('auth_logout.html') +    return render_template("auth_logout.html") + -@app.route('/auth/account') +@app.route("/auth/account")  @login_required  def auth_account():      # auth check on account page -    user_api = auth_api(session['api_token']) +    user_api = auth_api(session["api_token"])      resp = user_api.auth_check() -    assert(resp.success) -    editor = user_api.get_editor(session['editor']['editor_id']) -    session['editor'] = editor.to_dict() +    assert resp.success +    editor = user_api.get_editor(session["editor"]["editor_id"]) +    session["editor"] = editor.to_dict()      load_user(editor.editor_id) -    return render_template('auth_account.html') +    return render_template("auth_account.html") + -@app.route('/auth/wikipedia/auth') +@app.route("/auth/wikipedia/auth")  def wp_oauth_rewrite():      """      This is a dirty hack to rewrite '/auth/wikipedia/auth' to '/auth/wikipedia/oauth-callback'      """      return redirect( -        (b"/auth/wikipedia/oauth-callback?" + request.query_string).decode('utf-8'), +        (b"/auth/wikipedia/oauth-callback?" + request.query_string).decode("utf-8"),          307,      ) -@app.route('/auth/wikipedia/finish-login') + +@app.route("/auth/wikipedia/finish-login")  def wp_oauth_finish_login():      wp_username = mwoauth.get_current_user(cached=True) -    assert(wp_username) +    assert wp_username      return handle_wmoauth(wp_username)  ### Static Routes ########################################################### +  @app.errorhandler(404)  def page_not_found(e): -    return render_template('404.html'), 404 +    return render_template("404.html"), 404 +  @app.errorhandler(401)  @app.errorhandler(403)  def page_not_authorized(e): -    return render_template('403.html'), 403 +    return render_template("403.html"), 403 +  @app.errorhandler(405)  def page_method_not_allowed(e): -    return render_template('405.html'), 405 +    return render_template("405.html"), 405 +  @app.errorhandler(400)  def page_bad_request(e): -    return render_template('400.html', err=e), 400 +    return render_template("400.html", err=e), 400 +  @app.errorhandler(409)  def page_edit_conflict(e): -    return render_template('409.html'), 409 +    return render_template("409.html"), 409 +  @app.errorhandler(500)  def page_server_error(e):      app.log.error(e) -    return render_template('500.html'), 500 +    return render_template("500.html"), 500 +  @app.errorhandler(502)  @app.errorhandler(503)  @app.errorhandler(504)  def page_server_down(e):      app.log.error(e) -    return render_template('503.html'), 503 +    return render_template("503.html"), 503 +  @app.errorhandler(ApiException)  def page_fatcat_api_error(ae): @@ -1214,47 +1442,54 @@ def page_fatcat_api_error(ae):          return page_edit_conflict(ae)      try:          json_body = json.loads(ae.body) -        ae.error_name = json_body.get('error') -        ae.message = json_body.get('message') +        ae.error_name = json_body.get("error") +        ae.message = json_body.get("message")      except ValueError:          pass -    return render_template('api_error.html', api_error=ae), ae.status +    return render_template("api_error.html", api_error=ae), ae.status +  @app.errorhandler(ApiValueError)  def page_fatcat_api_value_error(ae):      ae.status = 400      ae.error_name = "ValueError"      ae.message = str(ae) -    return render_template('api_error.html', api_error=ae), 400 +    return render_template("api_error.html", api_error=ae), 400 +  @app.errorhandler(CSRFError)  def page_csrf_error(e): -    return render_template('csrf_error.html', reason=e.description), 400 +    return render_template("csrf_error.html", reason=e.description), 400 + -@app.route('/', methods=['GET']) +@app.route("/", methods=["GET"])  def page_home(): -    return render_template('home.html') +    return render_template("home.html") -@app.route('/about', methods=['GET']) + +@app.route("/about", methods=["GET"])  def page_about(): -    return render_template('about.html') +    return render_template("about.html") + -@app.route('/rfc', methods=['GET']) +@app.route("/rfc", methods=["GET"])  def page_rfc(): -    return render_template('rfc.html') +    return render_template("rfc.html") + -@app.route('/robots.txt', methods=['GET']) +@app.route("/robots.txt", methods=["GET"])  def page_robots_txt(): -    if app.config['FATCAT_DOMAIN'] == "fatcat.wiki": +    if app.config["FATCAT_DOMAIN"] == "fatcat.wiki":          robots_path = "robots.txt"      else:          robots_path = "robots.deny_all.txt" -    return send_from_directory(os.path.join(app.root_path, 'static'), -                               robots_path, -                               mimetype='text/plain') +    return send_from_directory( +        os.path.join(app.root_path, "static"), robots_path, mimetype="text/plain" +    ) + -@app.route('/sitemap.xml', methods=['GET']) +@app.route("/sitemap.xml", methods=["GET"])  def page_sitemap_xml(): -    return send_from_directory(os.path.join(app.root_path, 'static'), -                               "sitemap.xml", -                               mimetype='text/xml') +    return send_from_directory( +        os.path.join(app.root_path, "static"), "sitemap.xml", mimetype="text/xml" +    ) diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py index 73781016..5fc3f614 100644 --- a/python/fatcat_web/search.py +++ b/python/fatcat_web/search.py @@ -1,4 +1,3 @@ -  """  Helpers for doing elasticsearch queries (used in the web interface; not part of  the formal API) @@ -17,7 +16,6 @@ from fatcat_web import app  class FatcatSearchError(Exception): -      def __init__(self, status_code: int, name: str, description: str = None):          if status_code == "N/A":              status_code = 503 @@ -25,6 +23,7 @@ class FatcatSearchError(Exception):          self.name = name          self.description = description +  @dataclass  class ReleaseQuery:      q: Optional[str] = None @@ -35,31 +34,32 @@ class ReleaseQuery:      recent: bool = False      @classmethod -    def from_args(cls, args) -> 'ReleaseQuery': +    def from_args(cls, args) -> "ReleaseQuery": -        query_str = args.get('q') or '*' +        query_str = args.get("q") or "*" -        container_id = args.get('container_id') +        container_id = args.get("container_id")          # TODO: as filter, not in query string          if container_id:              query_str += ' container_id:"{}"'.format(container_id)          # TODO: where are container_issnl queries actually used? -        issnl = args.get('container_issnl') +        issnl = args.get("container_issnl")          if issnl and query_str:              query_str += ' container_issnl:"{}"'.format(issnl) -        offset = args.get('offset', '0') +        offset = args.get("offset", "0")          offset = max(0, int(offset)) if offset.isnumeric() else 0          return ReleaseQuery(              q=query_str,              offset=offset, -            fulltext_only=bool(args.get('fulltext_only')), +            fulltext_only=bool(args.get("fulltext_only")),              container_id=container_id, -            recent=bool(args.get('recent')), +            recent=bool(args.get("recent")),          ) +  @dataclass  class GenericQuery:      q: Optional[str] = None @@ -67,11 +67,11 @@ class GenericQuery:      offset: Optional[int] = None      @classmethod -    def from_args(cls, args) -> 'GenericQuery': -        query_str = args.get('q') +    def from_args(cls, args) -> "GenericQuery": +        query_str = args.get("q")          if not query_str: -            query_str = '*' -        offset = args.get('offset', '0') +            query_str = "*" +        offset = args.get("offset", "0")          offset = max(0, int(offset)) if offset.isnumeric() else 0          return GenericQuery( @@ -79,6 +79,7 @@ class GenericQuery:              offset=offset,          ) +  @dataclass  class SearchHits:      count_returned: int @@ -89,6 +90,7 @@ class SearchHits:      query_time_ms: int      results: List[Any] +  def _hits_total_int(val: Any) -> int:      """      Compatibility hack between ES 6.x and 7.x. In ES 6x, total is returned as @@ -97,7 +99,7 @@ def _hits_total_int(val: Any) -> int:      if isinstance(val, int):          return val      else: -        return int(val['value']) +        return int(val["value"])  def results_to_dict(response: elasticsearch_dsl.response.Response) -> List[dict]: @@ -121,6 +123,7 @@ def results_to_dict(response: elasticsearch_dsl.response.Response) -> List[dict]                  h[key] = h[key].encode("utf8", "ignore").decode("utf8")      return results +  def wrap_es_execution(search: Search) -> Any:      """      Executes a Search object, and converts various ES error types into @@ -146,6 +149,7 @@ def wrap_es_execution(search: Search) -> Any:          raise FatcatSearchError(e.status_code, str(e.error), description)      return resp +  def agg_to_dict(agg) -> dict:      """      Takes a simple term aggregation result (with buckets) and returns a simple @@ -157,14 +161,13 @@ def agg_to_dict(agg) -> dict:      for bucket in agg.buckets:          result[bucket.key] = bucket.doc_count      if agg.sum_other_doc_count: -        result['_other'] = agg.sum_other_doc_count +        result["_other"] = agg.sum_other_doc_count      return result -def do_container_search( -    query: GenericQuery, deep_page_limit: int = 2000 -) -> SearchHits: -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_CONTAINER_INDEX']) +def do_container_search(query: GenericQuery, deep_page_limit: int = 2000) -> SearchHits: + +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_CONTAINER_INDEX"])      search = search.query(          "query_string", @@ -199,11 +202,10 @@ def do_container_search(          results=results,      ) -def do_release_search( -    query: ReleaseQuery, deep_page_limit: int = 2000 -) -> SearchHits: -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +def do_release_search(query: ReleaseQuery, deep_page_limit: int = 2000) -> SearchHits: + +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      # availability filters      if query.fulltext_only: @@ -240,7 +242,11 @@ def do_release_search(      search = search.query(          "boosting", -        positive=Q("bool", must=basic_biblio, should=[has_fulltext],), +        positive=Q( +            "bool", +            must=basic_biblio, +            should=[has_fulltext], +        ),          negative=poor_metadata,          negative_boost=0.5,      ) @@ -260,9 +266,13 @@ def do_release_search(      for h in results:          # Ensure 'contrib_names' is a list, not a single string -        if type(h['contrib_names']) is not list: -            h['contrib_names'] = [h['contrib_names'], ] -        h['contrib_names'] = [name.encode('utf8', 'ignore').decode('utf8') for name in h['contrib_names']] +        if type(h["contrib_names"]) is not list: +            h["contrib_names"] = [ +                h["contrib_names"], +            ] +        h["contrib_names"] = [ +            name.encode("utf8", "ignore").decode("utf8") for name in h["contrib_names"] +        ]      return SearchHits(          count_returned=len(results), @@ -274,6 +284,7 @@ def do_release_search(          results=results,      ) +  def get_elastic_container_random_releases(ident: str, limit=5) -> dict:      """      Returns a list of releases from the container. @@ -281,16 +292,16 @@ def get_elastic_container_random_releases(ident: str, limit=5) -> dict:      assert limit > 0 and limit <= 100 -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search = search.query( -        'bool', +        "bool",          must=[ -            Q('term', container_id=ident), -            Q('range', release_year={ "lte": datetime.datetime.today().year }), -        ] +            Q("term", container_id=ident), +            Q("range", release_year={"lte": datetime.datetime.today().year}), +        ],      ) -    search = search.sort('-in_web', '-release_date') -    search = search[:int(limit)] +    search = search.sort("-in_web", "-release_date") +    search = search[: int(limit)]      search = search.params(request_cache=True)      # not needed: search = search.params(track_total_hits=True) @@ -299,6 +310,7 @@ def get_elastic_container_random_releases(ident: str, limit=5) -> dict:      return results +  def get_elastic_entity_stats() -> dict:      """      TODO: files, filesets, webcaptures (no schema yet) @@ -312,11 +324,11 @@ def get_elastic_entity_stats() -> dict:      stats = {}      # release totals -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search.aggs.bucket( -        'release_ref_count', -        'sum', -        field='ref_count', +        "release_ref_count", +        "sum", +        field="ref_count",      )      search = search[:0]  # pylint: disable=unsubscriptable-object @@ -324,15 +336,15 @@ def get_elastic_entity_stats() -> dict:      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search) -    stats['release'] = { +    stats["release"] = {          "total": _hits_total_int(resp.hits.total),          "refs_total": int(resp.aggregations.release_ref_count.value),      }      # paper counts -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search = search.query( -        'terms', +        "terms",          release_type=[              "article-journal",              "paper-conference", @@ -341,17 +353,21 @@ def get_elastic_entity_stats() -> dict:          ],      )      search.aggs.bucket( -        'paper_like', -        'filters', +        "paper_like", +        "filters",          filters={ -            "in_web": { "term": { "in_web": "true" } }, -            "is_oa": { "term": { "is_oa": "true" } }, -            "in_kbart": { "term": { "in_kbart": "true" } }, -            "in_web_not_kbart": { "bool": { "filter": [ -                { "term": { "in_web": "true" } }, -                { "term": { "in_kbart": "false" } }, -            ]}}, -        } +            "in_web": {"term": {"in_web": "true"}}, +            "is_oa": {"term": {"is_oa": "true"}}, +            "in_kbart": {"term": {"in_kbart": "true"}}, +            "in_web_not_kbart": { +                "bool": { +                    "filter": [ +                        {"term": {"in_web": "true"}}, +                        {"term": {"in_kbart": "false"}}, +                    ] +                } +            }, +        },      )      search = search[:0] @@ -359,35 +375,36 @@ def get_elastic_entity_stats() -> dict:      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search)      buckets = resp.aggregations.paper_like.buckets -    stats['papers'] = { -        'total': _hits_total_int(resp.hits.total), -        'in_web': buckets.in_web.doc_count, -        'is_oa': buckets.is_oa.doc_count, -        'in_kbart': buckets.in_kbart.doc_count, -        'in_web_not_kbart': buckets.in_web_not_kbart.doc_count, +    stats["papers"] = { +        "total": _hits_total_int(resp.hits.total), +        "in_web": buckets.in_web.doc_count, +        "is_oa": buckets.is_oa.doc_count, +        "in_kbart": buckets.in_kbart.doc_count, +        "in_web_not_kbart": buckets.in_web_not_kbart.doc_count,      }      # container counts -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_CONTAINER_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_CONTAINER_INDEX"])      search.aggs.bucket( -        'release_ref_count', -        'sum', -        field='ref_count', +        "release_ref_count", +        "sum", +        field="ref_count",      )      search = search[:0]  # pylint: disable=unsubscriptable-object      search = search.params(request_cache=True)      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search) -    stats['container'] = { +    stats["container"] = {          "total": _hits_total_int(resp.hits.total),      }      return stats +  def get_elastic_search_coverage(query: ReleaseQuery) -> dict: -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search = search.query(          "query_string",          query=query.q, @@ -398,10 +415,10 @@ def get_elastic_search_coverage(query: ReleaseQuery) -> dict:          fields=["biblio"],      )      search.aggs.bucket( -        'preservation', -        'terms', -        field='preservation', -        missing='_unknown', +        "preservation", +        "terms", +        field="preservation", +        missing="_unknown",      )      if query.recent:          date_today = datetime.date.today() @@ -416,21 +433,24 @@ def get_elastic_search_coverage(query: ReleaseQuery) -> dict:      resp = wrap_es_execution(search)      preservation_bucket = agg_to_dict(resp.aggregations.preservation) -    preservation_bucket['total'] = _hits_total_int(resp.hits.total) -    for k in ('bright', 'dark', 'shadows_only', 'none'): +    preservation_bucket["total"] = _hits_total_int(resp.hits.total) +    for k in ("bright", "dark", "shadows_only", "none"):          if k not in preservation_bucket:              preservation_bucket[k] = 0 -    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: -        preservation_bucket['none'] += preservation_bucket['shadows_only'] -        preservation_bucket['shadows_only'] = 0 +    if app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]: +        preservation_bucket["none"] += preservation_bucket["shadows_only"] +        preservation_bucket["shadows_only"] = 0      stats = { -        'total': _hits_total_int(resp.hits.total), -        'preservation': preservation_bucket, +        "total": _hits_total_int(resp.hits.total), +        "preservation": preservation_bucket,      }      return stats -def get_elastic_container_stats(ident, issnl=None, es_client=None, es_index=None, merge_shadows=None): + +def get_elastic_container_stats( +    ident, issnl=None, es_client=None, es_index=None, merge_shadows=None +):      """      Returns dict:          ident @@ -444,41 +464,41 @@ def get_elastic_container_stats(ident, issnl=None, es_client=None, es_index=None      if not es_client:          es_client = app.es_client      if not es_index: -        es_index = app.config['ELASTICSEARCH_RELEASE_INDEX'] +        es_index = app.config["ELASTICSEARCH_RELEASE_INDEX"]      if merge_shadows is None: -        merge_shadows = app.config['FATCAT_MERGE_SHADOW_PRESERVATION'] +        merge_shadows = app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]      search = Search(using=es_client, index=es_index)      search = search.query( -        'term', +        "term",          container_id=ident,      )      search.aggs.bucket( -        'container_stats', -        'filters', +        "container_stats", +        "filters",          filters={              "in_web": { -                "term": { "in_web": True }, +                "term": {"in_web": True},              },              "in_kbart": { -                "term": { "in_kbart": True }, +                "term": {"in_kbart": True},              },              "is_preserved": { -                "term": { "is_preserved": True }, +                "term": {"is_preserved": True},              },          },      )      search.aggs.bucket( -        'preservation', -        'terms', -        field='preservation', -        missing='_unknown', +        "preservation", +        "terms", +        field="preservation", +        missing="_unknown",      )      search.aggs.bucket( -        'release_type', -        'terms', -        field='release_type', -        missing='_unknown', +        "release_type", +        "terms", +        field="release_type", +        missing="_unknown",      )      search = search[:0] @@ -489,27 +509,28 @@ def get_elastic_container_stats(ident, issnl=None, es_client=None, es_index=None      container_stats = resp.aggregations.container_stats.buckets      preservation_bucket = agg_to_dict(resp.aggregations.preservation) -    preservation_bucket['total'] = _hits_total_int(resp.hits.total) -    for k in ('bright', 'dark', 'shadows_only', 'none'): +    preservation_bucket["total"] = _hits_total_int(resp.hits.total) +    for k in ("bright", "dark", "shadows_only", "none"):          if k not in preservation_bucket:              preservation_bucket[k] = 0      if merge_shadows: -        preservation_bucket['none'] += preservation_bucket['shadows_only'] -        preservation_bucket['shadows_only'] = 0 +        preservation_bucket["none"] += preservation_bucket["shadows_only"] +        preservation_bucket["shadows_only"] = 0      release_type_bucket = agg_to_dict(resp.aggregations.release_type)      stats = { -        'ident': ident, -        'issnl': issnl, -        'total': _hits_total_int(resp.hits.total), -        'in_web': container_stats['in_web']['doc_count'], -        'in_kbart': container_stats['in_kbart']['doc_count'], -        'is_preserved': container_stats['is_preserved']['doc_count'], -        'preservation': preservation_bucket, -        'release_type': release_type_bucket, +        "ident": ident, +        "issnl": issnl, +        "total": _hits_total_int(resp.hits.total), +        "in_web": container_stats["in_web"]["doc_count"], +        "in_kbart": container_stats["in_kbart"]["doc_count"], +        "is_preserved": container_stats["is_preserved"]["doc_count"], +        "preservation": preservation_bucket, +        "release_type": release_type_bucket,      }      return stats +  def get_elastic_container_histogram_legacy(ident) -> List:      """      Fetches a stacked histogram of {year, in_ia}. This is for the older style @@ -522,48 +543,58 @@ def get_elastic_container_histogram_legacy(ident) -> List:          (year, in_ia, count)      """ -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search = search.query( -        'bool', +        "bool",          must=[ -            Q("range", release_year={ -                "gte": datetime.datetime.today().year - 499, -                "lte": datetime.datetime.today().year, -            }), +            Q( +                "range", +                release_year={ +                    "gte": datetime.datetime.today().year - 499, +                    "lte": datetime.datetime.today().year, +                }, +            ),          ],          filter=[ -            Q("bool", minimum_should_match=1, should=[ -                Q("match", container_id=ident), -            ]), +            Q( +                "bool", +                minimum_should_match=1, +                should=[ +                    Q("match", container_id=ident), +                ], +            ),          ],      )      search.aggs.bucket( -        'year_in_ia', -        'composite', +        "year_in_ia", +        "composite",          size=1000,          sources=[ -            {"year": { -                "histogram": { -                    "field": "release_year", -                    "interval": 1, -                }, -            }}, -            {"in_ia": { -                "terms": { -                    "field": "in_ia", -                }, -            }}, +            { +                "year": { +                    "histogram": { +                        "field": "release_year", +                        "interval": 1, +                    }, +                } +            }, +            { +                "in_ia": { +                    "terms": { +                        "field": "in_ia", +                    }, +                } +            },          ],      )      search = search[:0] -    search = search.params(request_cache='true') +    search = search.params(request_cache="true")      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search)      buckets = resp.aggregations.year_in_ia.buckets -    vals = [(int(h['key']['year']), h['key']['in_ia'], h['doc_count']) -            for h in buckets] +    vals = [(int(h["key"]["year"]), h["key"]["in_ia"], h["doc_count"]) for h in buckets]      vals = sorted(vals)      return vals @@ -580,7 +611,7 @@ def get_elastic_preservation_by_year(query) -> List[dict]:          {year (int), bright (int), dark (int), shadows_only (int), none (int)}      """ -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      if query.q not in [None, "*"]:          search = search.query(              "query_string", @@ -607,41 +638,47 @@ def get_elastic_preservation_by_year(query) -> List[dict]:      )      search.aggs.bucket( -        'year_preservation', -        'composite', +        "year_preservation", +        "composite",          size=1500,          sources=[ -            {"year": { -                "histogram": { -                    "field": "release_year", -                    "interval": 1, -                }, -            }}, -            {"preservation": { -                "terms": { -                    "field": "preservation", -                }, -            }}, +            { +                "year": { +                    "histogram": { +                        "field": "release_year", +                        "interval": 1, +                    }, +                } +            }, +            { +                "preservation": { +                    "terms": { +                        "field": "preservation", +                    }, +                } +            },          ],      )      search = search[:0] -    search = search.params(request_cache='true') +    search = search.params(request_cache="true")      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search)      buckets = resp.aggregations.year_preservation.buckets -    year_nums = set([int(h['key']['year']) for h in buckets]) +    year_nums = set([int(h["key"]["year"]) for h in buckets])      year_dicts = dict()      if year_nums: -        for num in range(min(year_nums), max(year_nums)+1): +        for num in range(min(year_nums), max(year_nums) + 1):              year_dicts[num] = dict(year=num, bright=0, dark=0, shadows_only=0, none=0)          for row in buckets: -            year_dicts[int(row['key']['year'])][row['key']['preservation']] = int(row['doc_count']) -    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: +            year_dicts[int(row["key"]["year"])][row["key"]["preservation"]] = int( +                row["doc_count"] +            ) +    if app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]:          for k in year_dicts.keys(): -            year_dicts[k]['none'] += year_dicts[k]['shadows_only'] -            year_dicts[k]['shadows_only'] = 0 -    return sorted(year_dicts.values(), key=lambda x: x['year']) +            year_dicts[k]["none"] += year_dicts[k]["shadows_only"] +            year_dicts[k]["shadows_only"] = 0 +    return sorted(year_dicts.values(), key=lambda x: x["year"])  def get_elastic_preservation_by_date(query) -> List[dict]: @@ -656,7 +693,7 @@ def get_elastic_preservation_by_date(query) -> List[dict]:          {date (str), bright (int), dark (int), shadows_only (int), none (int)}      """ -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      if query.q not in [None, "*"]:          search = search.query(              "query_string", @@ -678,32 +715,37 @@ def get_elastic_preservation_by_date(query) -> List[dict]:      start_date = date_today - datetime.timedelta(days=60)      end_date = date_today + datetime.timedelta(days=1)      search = search.filter( -        "range", release_date=dict( +        "range", +        release_date=dict(              gte=str(start_date),              lte=str(end_date), -        ) +        ),      )      search.aggs.bucket( -        'date_preservation', -        'composite', +        "date_preservation", +        "composite",          size=1500,          sources=[ -            {"date": { -                "histogram": { -                    "field": "release_date", -                    "interval": 1, -                }, -            }}, -            {"preservation": { -                "terms": { -                    "field": "preservation", -                }, -            }}, +            { +                "date": { +                    "histogram": { +                        "field": "release_date", +                        "interval": 1, +                    }, +                } +            }, +            { +                "preservation": { +                    "terms": { +                        "field": "preservation", +                    }, +                } +            },          ],      )      search = search[:0] -    search = search.params(request_cache='true') +    search = search.params(request_cache="true")      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search) @@ -711,15 +753,18 @@ def get_elastic_preservation_by_date(query) -> List[dict]:      date_dicts = dict()      this_date = start_date      while this_date <= end_date: -        date_dicts[str(this_date)] = dict(date=str(this_date), bright=0, dark=0, shadows_only=0, none=0) +        date_dicts[str(this_date)] = dict( +            date=str(this_date), bright=0, dark=0, shadows_only=0, none=0 +        )          this_date = this_date + datetime.timedelta(days=1)      for row in buckets: -        date_dicts[row['key']['date'][0:10]][row['key']['preservation']] = int(row['doc_count']) -    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: +        date_dicts[row["key"]["date"][0:10]][row["key"]["preservation"]] = int(row["doc_count"]) +    if app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]:          for k in date_dicts.keys(): -            date_dicts[k]['none'] += date_dicts[k]['shadows_only'] -            date_dicts[k]['shadows_only'] = 0 -    return sorted(date_dicts.values(), key=lambda x: x['date']) +            date_dicts[k]["none"] += date_dicts[k]["shadows_only"] +            date_dicts[k]["shadows_only"] = 0 +    return sorted(date_dicts.values(), key=lambda x: x["date"]) +  def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict]:      """ @@ -733,52 +778,64 @@ def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict          {year (int), bright (int), dark (int), shadows_only (int), none (int)}      """ -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      search = search.query( -        'bool', +        "bool",          filter=[ -            Q("bool", must=[ -                Q("match", container_id=container_id), -                Q("exists", field="volume"), -            ]), +            Q( +                "bool", +                must=[ +                    Q("match", container_id=container_id), +                    Q("exists", field="volume"), +                ], +            ),          ],      )      search.aggs.bucket( -        'volume_preservation', -        'composite', +        "volume_preservation", +        "composite",          size=1500,          sources=[ -            {"volume": { -                "terms": { -                    "field": "volume", -                }, -            }}, -            {"preservation": { -                "terms": { -                    "field": "preservation", -                }, -            }}, +            { +                "volume": { +                    "terms": { +                        "field": "volume", +                    }, +                } +            }, +            { +                "preservation": { +                    "terms": { +                        "field": "preservation", +                    }, +                } +            },          ],      )      search = search[:0] -    search = search.params(request_cache='true') +    search = search.params(request_cache="true")      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search)      buckets = resp.aggregations.volume_preservation.buckets -    volume_nums = set([int(h['key']['volume']) for h in buckets if h['key']['volume'].isdigit()]) +    volume_nums = set( +        [int(h["key"]["volume"]) for h in buckets if h["key"]["volume"].isdigit()] +    )      volume_dicts = dict()      if volume_nums: -        for num in range(min(volume_nums), max(volume_nums)+1): +        for num in range(min(volume_nums), max(volume_nums) + 1):              volume_dicts[num] = dict(volume=num, bright=0, dark=0, shadows_only=0, none=0)          for row in buckets: -            if row['key']['volume'].isdigit(): -                volume_dicts[int(row['key']['volume'])][row['key']['preservation']] = int(row['doc_count']) -    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: +            if row["key"]["volume"].isdigit(): +                volume_dicts[int(row["key"]["volume"])][row["key"]["preservation"]] = int( +                    row["doc_count"] +                ) +    if app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]:          for k in volume_dicts.keys(): -            volume_dicts[k]['none'] += volume_dicts[k]['shadows_only'] -            volume_dicts[k]['shadows_only'] = 0 -    return sorted(volume_dicts.values(), key=lambda x: x['volume']) +            volume_dicts[k]["none"] += volume_dicts[k]["shadows_only"] +            volume_dicts[k]["shadows_only"] = 0 +    return sorted(volume_dicts.values(), key=lambda x: x["volume"]) +  def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:      """ @@ -789,7 +846,7 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:          {year (int), bright (int), dark (int), shadows_only (int), none (int)}      """ -    search = Search(using=app.es_client, index=app.config['ELASTICSEARCH_RELEASE_INDEX']) +    search = Search(using=app.es_client, index=app.config["ELASTICSEARCH_RELEASE_INDEX"])      if query.q not in [None, "*"]:          search = search.query(              "query_string", @@ -804,11 +861,14 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:          )      if query.container_id:          search = search.query( -            'bool', +            "bool",              filter=[ -                Q("bool", must=[ -                    Q("match", container_id=query.container_id), -                ]), +                Q( +                    "bool", +                    must=[ +                        Q("match", container_id=query.container_id), +                    ], +                ),              ],          )      if query.recent: @@ -817,39 +877,45 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:          end_date = str(date_today + datetime.timedelta(days=1))          search = search.filter("range", release_date=dict(gte=start_date, lte=end_date))      search.aggs.bucket( -        'type_preservation', -        'composite', +        "type_preservation", +        "composite",          size=1500,          sources=[ -            {"release_type": { -                "terms": { -                    "field": "release_type", -                }, -            }}, -            {"preservation": { -                "terms": { -                    "field": "preservation", -                }, -            }}, +            { +                "release_type": { +                    "terms": { +                        "field": "release_type", +                    }, +                } +            }, +            { +                "preservation": { +                    "terms": { +                        "field": "preservation", +                    }, +                } +            },          ],      )      search = search[:0] -    search = search.params(request_cache='true') +    search = search.params(request_cache="true")      search = search.params(track_total_hits=True)      resp = wrap_es_execution(search)      buckets = resp.aggregations.type_preservation.buckets -    type_set = set([h['key']['release_type'] for h in buckets]) +    type_set = set([h["key"]["release_type"] for h in buckets])      type_dicts = dict()      for k in type_set:          type_dicts[k] = dict(release_type=k, bright=0, dark=0, shadows_only=0, none=0, total=0)      for row in buckets: -        type_dicts[row['key']['release_type']][row['key']['preservation']] = int(row['doc_count']) +        type_dicts[row["key"]["release_type"]][row["key"]["preservation"]] = int( +            row["doc_count"] +        )      for k in type_set: -        for p in ('bright', 'dark', 'shadows_only', 'none'): -            type_dicts[k]['total'] += type_dicts[k][p] -    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']: +        for p in ("bright", "dark", "shadows_only", "none"): +            type_dicts[k]["total"] += type_dicts[k][p] +    if app.config["FATCAT_MERGE_SHADOW_PRESERVATION"]:          for k in type_set: -            type_dicts[k]['none'] += type_dicts[k]['shadows_only'] -            type_dicts[k]['shadows_only'] = 0 -    return sorted(type_dicts.values(), key=lambda x: x['total'], reverse=True) +            type_dicts[k]["none"] += type_dicts[k]["shadows_only"] +            type_dicts[k]["shadows_only"] = 0 +    return sorted(type_dicts.values(), key=lambda x: x["total"], reverse=True) diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py index c15fefa4..229c2761 100644 --- a/python/fatcat_web/web_config.py +++ b/python/fatcat_web/web_config.py @@ -1,4 +1,3 @@ -  """  Default configuration for fatcat web interface (Flask application). @@ -16,26 +15,43 @@ import raven  basedir = os.path.abspath(os.path.dirname(__file__)) +  class Config(object): -    GIT_REVISION = subprocess.check_output(["git", "describe", "--tags", "--long", "--always"]).strip().decode('utf-8') +    GIT_REVISION = ( +        subprocess.check_output(["git", "describe", "--tags", "--long", "--always"]) +        .strip() +        .decode("utf-8") +    )      # This is, effectively, the QA/PROD flag      FATCAT_DOMAIN = os.environ.get("FATCAT_DOMAIN", default="dev.fatcat.wiki")      FATCAT_API_AUTH_TOKEN = os.environ.get("FATCAT_API_AUTH_TOKEN", default=None) -    FATCAT_API_HOST = os.environ.get("FATCAT_API_HOST", default=f"https://api.{FATCAT_DOMAIN}/v0") +    FATCAT_API_HOST = os.environ.get( +        "FATCAT_API_HOST", default=f"https://api.{FATCAT_DOMAIN}/v0" +    )      public_host_default = f"https://api.{FATCAT_DOMAIN}/v0"      if FATCAT_DOMAIN == "dev.fatcat.wiki":          public_host_default = FATCAT_API_HOST -    FATCAT_PUBLIC_API_HOST = os.environ.get("FATCAT_PUBLIC_API_HOST", default=public_host_default) +    FATCAT_PUBLIC_API_HOST = os.environ.get( +        "FATCAT_PUBLIC_API_HOST", default=public_host_default +    )      # can set this to https://search.fatcat.wiki for some experimentation -    ELASTICSEARCH_BACKEND = os.environ.get("ELASTICSEARCH_BACKEND", default="http://localhost:9200") -    ELASTICSEARCH_RELEASE_INDEX = os.environ.get("ELASTICSEARCH_RELEASE_INDEX", default="fatcat_release") -    ELASTICSEARCH_CONTAINER_INDEX = os.environ.get("ELASTICSEARCH_CONTAINER_INDEX", default="fatcat_container") +    ELASTICSEARCH_BACKEND = os.environ.get( +        "ELASTICSEARCH_BACKEND", default="http://localhost:9200" +    ) +    ELASTICSEARCH_RELEASE_INDEX = os.environ.get( +        "ELASTICSEARCH_RELEASE_INDEX", default="fatcat_release" +    ) +    ELASTICSEARCH_CONTAINER_INDEX = os.environ.get( +        "ELASTICSEARCH_CONTAINER_INDEX", default="fatcat_container" +    )      # for save-paper-now. set to None if not configured, so we don't display forms/links      KAFKA_PIXY_ENDPOINT = os.environ.get("KAFKA_PIXY_ENDPOINT", default=None) or None -    KAFKA_SAVEPAPERNOW_TOPIC = os.environ.get("KAFKA_SAVEPAPERNOW_TOPIC", default="sandcrawler-dev.ingest-file-requests-priority") +    KAFKA_SAVEPAPERNOW_TOPIC = os.environ.get( +        "KAFKA_SAVEPAPERNOW_TOPIC", default="sandcrawler-dev.ingest-file-requests-priority" +    )      # for flask things, like session cookies      FLASK_SECRET_KEY = os.environ.get("FLASK_SECRET_KEY", default=None) @@ -59,11 +75,17 @@ class Config(object):      # analytics; used in production      ENABLE_GOATCOUNTER = bool(os.environ.get("ENABLE_GOATCOUNTER", default=False)) -    GOATCOUNTER_ENDPOINT = os.environ.get("GOATCOUNTER_ENDPOINT", default="https://goatcounter.fatcat.wiki/count") -    GOATCOUNTER_SCRIPT_URL = os.environ.get("GOATCOUNTER_SCRIPT_URL", default="https://goatcounter.fatcat.wiki/count.js") +    GOATCOUNTER_ENDPOINT = os.environ.get( +        "GOATCOUNTER_ENDPOINT", default="https://goatcounter.fatcat.wiki/count" +    ) +    GOATCOUNTER_SCRIPT_URL = os.environ.get( +        "GOATCOUNTER_SCRIPT_URL", default="https://goatcounter.fatcat.wiki/count.js" +    )      # controls granularity of "shadow_only" preservation category -    FATCAT_MERGE_SHADOW_PRESERVATION = os.environ.get("FATCAT_MERGE_SHADOW_PRESERVATION", default=False) +    FATCAT_MERGE_SHADOW_PRESERVATION = os.environ.get( +        "FATCAT_MERGE_SHADOW_PRESERVATION", default=False +    )      # CSRF on by default, but only for WTF forms (not, eg, search, lookups, GET      # forms) @@ -75,27 +97,27 @@ class Config(object):      if FATCAT_DOMAIN == "dev.fatcat.wiki":          # "Even more verbose" debug options -        #SQLALCHEMY_ECHO = True -        #DEBUG = True +        # SQLALCHEMY_ECHO = True +        # DEBUG = True          pass      else:          # protect cookies (which include API tokens)          SESSION_COOKIE_HTTPONLY = True          SESSION_COOKIE_SECURE = True -        SESSION_COOKIE_SAMESITE = 'Lax' -        PERMANENT_SESSION_LIFETIME = 2678400 # 31 days, in seconds +        SESSION_COOKIE_SAMESITE = "Lax" +        PERMANENT_SESSION_LIFETIME = 2678400  # 31 days, in seconds      try: -        GIT_RELEASE = raven.fetch_git_sha('..') +        GIT_RELEASE = raven.fetch_git_sha("..")      except Exception as e:          print("WARNING: couldn't set sentry git release automatically: " + str(e))          GIT_RELEASE = None      SENTRY_CONFIG = {          #'include_paths': ['fatcat_web', 'fatcat_openapi_client', 'fatcat_tools'], -        'enable-threads': True, # for uWSGI -        'release': GIT_RELEASE, -        'tags': { -            'fatcat_domain': FATCAT_DOMAIN, +        "enable-threads": True,  # for uWSGI +        "release": GIT_RELEASE, +        "tags": { +            "fatcat_domain": FATCAT_DOMAIN,          },      } | 
