Merge branch 'bnewbold-crude-auth'

Fixed a conflict in:
  python/fatcat_export.py

12 files changed, 446 insertions, 22 deletions

diff --git a/python/fatcat_web/__init__.py b/python/fatcat_web/__init__.py
index 3c790e7a..cd7af195 100644
--- a/python/fatcat_web/__init__.py
+++ b/python/fatcat_web/__init__.py
@@ -2,21 +2,47 @@
 from flask import Flask
 from flask_uuid import FlaskUUID
 from flask_debugtoolbar import DebugToolbarExtension
+from flask_login import LoginManager
+from authlib.flask.client import OAuth
+from loginpass import create_flask_blueprint, Gitlab
 from raven.contrib.flask import Sentry
-from web_config import Config
 import fatcat_client
 
+from fatcat_web.web_config import Config
+
 toolbar = DebugToolbarExtension()
 app = Flask(__name__)
 app.config.from_object(Config)
 toolbar = DebugToolbarExtension(app)
 FlaskUUID(app)
 
+login_manager = LoginManager()
+login_manager.init_app(app)
+login_manager.login_view = "/auth/login"
+oauth = OAuth(app)
+
 # Grabs sentry config from SENTRY_DSN environment variable
 sentry = Sentry(app)
 
 conf = fatcat_client.Configuration()
-conf.host = "http://localhost:9411/v0"
+conf.host = Config.FATCAT_API_HOST
 api = fatcat_client.DefaultApi(fatcat_client.ApiClient(conf))
 
-from fatcat_web import routes
+def auth_api(token):
+    conf = fatcat_client.Configuration()
+    conf.api_key["Authorization"] = token
+    conf.api_key_prefix["Authorization"] = "Bearer"
+    conf.host = Config.FATCAT_API_HOST
+    return fatcat_client.DefaultApi(fatcat_client.ApiClient(conf))
+
+if Config.FATCAT_API_AUTH_TOKEN:
+    print("Found and using privileged token (eg, for account signup)")
+    priv_api = auth_api(Config.FATCAT_API_AUTH_TOKEN)
+else:
+    print("No privileged token found")
+    priv_api = None
+
+from fatcat_web import routes, auth
+
+gitlab_bp = create_flask_blueprint(Gitlab, oauth, auth.handle_oauth)
+app.register_blueprint(gitlab_bp, url_prefix='/auth/gitlab')
diff --git a/python/fatcat_web/auth.py b/python/fatcat_web/auth.py
new file mode 100644
index 00000000..8035cbe5
--- /dev/null
+++ b/python/fatcat_web/auth.py
@@ -0,0 +1,141 @@
+
+from collections import namedtuple
+import requests
+import pymacaroons
+from flask import Flask, render_template, send_from_directory, request, \
+    url_for, abort, g, redirect, jsonify, session, flash
+from fatcat_web import login_manager, api, priv_api, Config
+from flask_login import logout_user, login_user, UserMixin
+import fatcat_client
+
+def handle_logout():
+    logout_user()
+    for k in ('editor', 'api_token'):
+        if k in session:
+            session.pop(k)
+    session.clear()
+
+def handle_token_login(token):
+    try:
+        m = pymacaroons.Macaroon.deserialize(token)
+    except pymacaroons.exceptions.MacaroonDeserializationException:
+        # TODO: what kind of Exceptions?
+        return abort(400)
+    # extract editor_id
+    editor_id = None
+    for caveat in m.first_party_caveats():
+        caveat = caveat.caveat_id
+        if caveat.startswith(b"editor_id = "):
+            editor_id = caveat[12:].decode('utf-8')
+    if not editor_id:
+        abort(400)
+    # fetch editor info
+    editor = api.get_editor(editor_id)
+    session.permanent = True
+    session['api_token'] = token
+    session['editor'] = editor.to_dict()
+    login_user(load_user(editor.editor_id))
+    return redirect("/auth/account")
+
+# This will need to login/signup via fatcatd API, then set token in session
+def handle_oauth(remote, token, user_info):
+    if user_info:
+        # fetch api login/signup using user_info
+        # ISS is basically the API url (though more formal in OIDC)
+        # SUB is the stable internal identifier for the user (not usually the username itself)
+        # TODO: should have the real sub here
+        # TODO: would be nicer to pass preferred_username for account creation
+        iss = remote.OAUTH_CONFIG['api_base_url']
+
+        # we reuse 'preferred_username' for account name auto-creation (but
+        # don't store it otherwise in the backend, at least currently). But i'm
+        # not sure all loginpass backends will set it
+        if user_info.get('preferred_username'):
+            preferred_username = user_info['preferred_username']
+        else:
+            preferred_username = user_info['sub']
+
+        params = fatcat_client.AuthOidc(remote.name, user_info['sub'], iss, user_info['preferred_username'])
+        # this call requires admin privs
+        (resp, http_status, http_headers) = priv_api.auth_oidc_with_http_info(params)
+        editor = resp.editor
+        api_token = resp.token
+
+        if http_status == 201:
+            flash("Welcome to Fatcat! An account has been created for you with a temporary username; you may wish to change it under account settings")
+            flash("You must use the same mechanism ({}) to login in the future".format(remote.name))
+        else:
+            flash("Welcome back!")
+
+        # write token and username to session
+        session.permanent = True
+        session['api_token'] = api_token
+        session['editor'] = editor.to_dict()
+
+        # call login_user(load_user(editor_id))
+        login_user(load_user(editor.editor_id))
+        return redirect("/auth/account")
+
+    # XXX: what should this actually be?
+    raise Exception("didn't receive OAuth user_info")
+
+def handle_ia_xauth(email, password):
+    resp = requests.post(Config.IA_XAUTH_URI,
+        params={'op': 'authenticate'},
+        json={
+            'version': '1',
+            'email': email,
+            'password': password,
+            'access': Config.IA_XAUTH_CLIENT_ID,
+            'secret': Config.IA_XAUTH_CLIENT_SECRET,
+        })
+    if resp.status_code == 401 or (not resp.json().get('success')):
+        flash("Internet Archive email/password didn't match: {}".format(resp.json()['values']['reason']))
+        return render_template('auth_ia_login.html', email=email), resp.status_code
+    elif resp.status_code != 200:
+        flash("Internet Archive login failed (internal error?)")
+        # TODO: log.warn
+        print("IA XAuth fail: {}".format(resp.content))
+        return render_template('auth_ia_login.html', email=email), resp.status_code
+
+    # Successful login; now fetch info...
+    resp = requests.post(Config.IA_XAUTH_URI,
+        params={'op': 'info'},
+        json={
+            'version': '1',
+            'email': email,
+            'access': Config.IA_XAUTH_CLIENT_ID,
+            'secret': Config.IA_XAUTH_CLIENT_SECRET,
+        })
+    if resp.status_code != 200:
+        flash("Internet Archive login failed (internal error?)")
+        # TODO: log.warn
+        print("IA XAuth fail: {}".format(resp.content))
+        return render_template('auth_ia_login.html', email=email), resp.status_code
+    ia_info = resp.json()['values']
+
+    # and pass off "as if" we did OAuth successfully
+    FakeOAuthRemote = namedtuple('FakeOAuthRemote', ['name', 'OAUTH_CONFIG'])
+    remote = FakeOAuthRemote(name='archive', OAUTH_CONFIG={'api_base_url': Config.IA_XAUTH_URI})
+    oauth_info = {
+        'preferred_username': ia_info['screenname'],
+        'iss': Config.IA_XAUTH_URI,
+        'sub': ia_info['itemname'],
+    }
+    return handle_oauth(remote, None, oauth_info)
+
+@login_manager.user_loader
+def load_user(editor_id):
+    # looks for extra info in session, and updates the user object with that.
+    # If session isn't loaded/valid, should return None
+    if (not session.get('editor')) or (not session.get('api_token')):
+        return None
+    editor = session['editor']
+    token = session['api_token']
+    user = UserMixin()
+    user.id = editor_id
+    user.editor_id = editor_id
+    user.username = editor['username']
+    user.token = token
+    return user
+
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index 998697bc..789d7bed 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -2,8 +2,10 @@
 import os
 import json
 from flask import Flask, render_template, send_from_directory, request, \
-    url_for, abort, g, redirect, jsonify, session
-from fatcat_web import app, api
+    url_for, abort, g, redirect, jsonify, session, flash
+from flask_login import login_required
+from fatcat_web import app, api, auth_api
+from fatcat_web.auth import handle_token_login, handle_logout, load_user, handle_ia_xauth
 from fatcat_client.rest import ApiException
 from fatcat_web.search import do_search
 
@@ -295,12 +297,6 @@ def work_view(ident):
         return render_template('deleted_entity.html', entity=entity)
     return render_template('work_view.html', work=entity, releases=releases)
 
-@app.route('/editgroup/current', methods=['GET'])
-def editgroup_current():
-    raise NotImplementedError
-    #eg = api.get_or_create_editgroup()
-    #return redirect('/editgroup/{}'.format(eg.id))
-
 @app.route('/editgroup/<ident>', methods=['GET'])
 def editgroup_view(ident):
     try:
@@ -327,6 +323,17 @@ def editor_changelog(ident):
     return render_template('editor_changelog.html', editor=editor,
         changelog_entries=changelog_entries)
 
+@app.route('/editor/<ident>/wip', methods=['GET'])
+def editor_wip(ident):
+    raise NotImplementedError
+    try:
+        editor = api.get_editor(ident)
+        entries = api.get_editor_wip(ident)
+    except ApiException as ae:
+        abort(ae.status)
+    return render_template('editor_changelog.html', editor=editor,
+        entries=entries)
+
 @app.route('/changelog', methods=['GET'])
 def changelog_view():
     try:
@@ -367,6 +374,61 @@ def search():
         return render_template('release_search.html', query=query, fulltext_only=fulltext_only)
 
 
+### Auth ####################################################################
+
+@app.route('/auth/login')
+def login():
+    # show the user a list of login options
+    return render_template('auth_login.html')
+
+@app.route('/auth/ia/login', methods=['GET', 'POST'])
+def ia_xauth_login():
+    if 'email' in request.form:
+        # if a login attempt...
+        return handle_ia_xauth(request.form.get('email'), request.form.get('password'))
+    # else show form
+    return render_template('auth_ia_login.html')
+
+@app.route('/auth/token_login', methods=['GET', 'POST'])
+def token_login():
+    # show the user a list of login options
+    if 'token' in request.args:
+        return handle_token_login(request.args.get('token'))
+    if 'token' in request.form:
+        return handle_token_login(request.form.get('token'))
+    return render_template('auth_token_login.html')
+
+@app.route('/auth/change_username', methods=['POST'])
+@login_required
+def change_username():
+    # show the user a list of login options
+    if not 'username' in request.form:
+        abort(400)
+    # on behalf of user...
+    user_api = auth_api(session['api_token'])
+    editor = user_api.get_editor(session['editor']['editor_id'])
+    editor.username = request.form['username']
+    editor = user_api.update_editor(editor.editor_id, editor)
+    # update our session
+    session['editor'] = editor.to_dict()
+    load_user(editor.editor_id)
+    flash("Username updated successfully")
+    return redirect('/auth/account')
+
+@app.route('/auth/logout')
+def logout():
+    handle_logout()
+    return render_template('auth_logout.html')
+
+@app.route('/auth/account')
+@login_required
+def auth_account():
+    editor = api.get_editor(session['editor']['editor_id'])
+    session['editor'] = editor.to_dict()
+    load_user(editor.editor_id)
+    return render_template('auth_account.html')
+
+
 ### Static Routes ###########################################################
 
 @app.errorhandler(404)
diff --git a/python/fatcat_web/templates/auth_account.html b/python/fatcat_web/templates/auth_account.html
new file mode 100644
index 00000000..57155722
--- /dev/null
+++ b/python/fatcat_web/templates/auth_account.html
@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+{% block body %}
+
+<h1>Your Account</h1>
+
+<p><b>Username:</b> <code>{{ current_user.username }}</code>
+<p><b>Editor Id:</b> <code><a href="/editor/{{ current_user.editor_id }}">{{ current_user.editor_id }}</a></code>
+
+<div>
+<p>Change username:
+<form class="" role="change_username" action="/auth/change_username" method="post">
+  <div class="ui form">
+      <div class="ui action input medium fluid">
+      <input type="text" name="username" value="{{ current_user.username }}" aria-label="account username">
+      <button class="ui button">Update</button>
+    </div>
+  </div>
+</form>
+</div>
+
+<p>In the future, you might be able to...
+<ul>
+  <li>Create a bot user
+  <li>Generate an API token
+</ul>
+
+{% endblock %}
diff --git a/python/fatcat_web/templates/auth_ia_login.html b/python/fatcat_web/templates/auth_ia_login.html
new file mode 100644
index 00000000..ebf08021
--- /dev/null
+++ b/python/fatcat_web/templates/auth_ia_login.html
@@ -0,0 +1,31 @@
+{% extends "base.html" %}
+{% block body %}
+<h1>Login with Internet Archive account</h1>
+
+<p>Warning: still experimental!
+
+<br>
+<br>
+<br>
+
+{% if current_user.is_authenticated %}
+  <div class="ui negative message">
+    <div class="header">You are already logged in!</div>
+    <p>You should logout first. Re-authenticating would be undefined behavior.
+  </div>
+{% else %}
+  <form class="" role="login" action="/auth/ia/login" method="post">
+    <div class="ui form">
+        <div class="ui input huge fluid">
+          <input type="email" placeholder="user@domain.tdl..." name="email" {% if email %}value="{{ email }}"{% endif %} aria-label="email for login">
+        </div>
+        <div class="ui action input huge fluid">
+          <input type="password" placeholder="password" name="password" aria-label="internet archive password">
+          <button class="ui button">Login</button>
+        </div>
+      </div>
+    </div>
+  </form>
+{% endif %}
+
+{% endblock %}
diff --git a/python/fatcat_web/templates/auth_login.html b/python/fatcat_web/templates/auth_login.html
new file mode 100644
index 00000000..9ccae816
--- /dev/null
+++ b/python/fatcat_web/templates/auth_login.html
@@ -0,0 +1,18 @@
+{% extends "base.html" %}
+{% block body %}
+<h1>Login</h1>
+
+<p>via OAuth / OpenID Connect:
+<ul>
+  <li><a href="/auth/gitlab/login">gitlab.com</a>
+  <li><strike><a href="/auth/google/login">google.com</a></strike>
+  <li><strike><a href="/auth/orcid/login">orcid.org</a></strike>
+</ul>
+
+<p>Other options...
+<ul>
+  <li><a href="/auth/token_login">Using auth token</a> (admin/operator)
+  <li><a href="/auth/ia/login">With Internet Archive account</a> (experimental)
+</ul>
+
+{% endblock %}
diff --git a/python/fatcat_web/templates/auth_logout.html b/python/fatcat_web/templates/auth_logout.html
new file mode 100644
index 00000000..819d42fe
--- /dev/null
+++ b/python/fatcat_web/templates/auth_logout.html
@@ -0,0 +1,8 @@
+{% extends "base.html" %}
+{% block body %}
+<h1>Logout</h1>
+
+<p>If you are seeing this page, you are now logged out.
+
+<p>Use the links above to return to the home page or log back in.
+{% endblock %}
diff --git a/python/fatcat_web/templates/auth_token_login.html b/python/fatcat_web/templates/auth_token_login.html
new file mode 100644
index 00000000..4c28f938
--- /dev/null
+++ b/python/fatcat_web/templates/auth_token_login.html
@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+{% block body %}
+<h1>Login with Token</h1>
+
+<p>This page is intended for operators and contingencies, not for general use. It
+allows editors (users) to use an existing token (macaroon) for authentication;
+a new web interface session and cookie are constructed using the token.
+
+<br>
+<br>
+<br>
+
+{% if current_user.is_authenticated %}
+  <div class="ui negative message">
+    <div class="header">You are already logged in!</div>
+    <p>You should logout first. Re-authenticating would be undefined behavior.
+  </div>
+{% else %}
+  <form class="" role="login" action="/auth/token_login" method="post">
+    <div class="ui form">
+        <div class="ui action input huge fluid">
+        <input type="password" placeholder="Your Fatcat API Auth Token..." name="token" value="{% if token %}{{ token }}{% endif %}" aria-label="login using token">
+        <button class="ui button">Login</button>
+      </div>
+    </div>
+  </form>
+{% endif %}
+
+{% endblock %}
diff --git a/python/fatcat_web/templates/base.html b/python/fatcat_web/templates/base.html
index 4b3b7e0b..cce841e5 100644
--- a/python/fatcat_web/templates/base.html
+++ b/python/fatcat_web/templates/base.html
@@ -29,23 +29,41 @@
           </div>
         </form>
       </div>
+{% if current_user.is_authenticated %}
       <div class="ui simple dropdown item">
-      demo-user <i class="dropdown icon"></i>
+      {{ current_user.username }} <i class="dropdown icon"></i>
         <div class="menu">
-          <a class="item" href="/editgroup/current"><i class="edit icon"></i>Edits in Progress</a>
-          <a class="item" href="/editor/aaaaaaaaaaaabkvkaaaaaaaaae/changelog"><i class="history icon"></i>History</a>
+          <a class="item" href="#"><i class="edit icon"></i>Edits in Progress</a>
+          <a class="item" href="/editor/{{ current_user.editor_id }}/changelog"><i class="history icon"></i>History</a>
           <div class="divider"></div>
-          <a class="item" href="/editor/aaaaaaaaaaaabkvkaaaaaaaaae"><i class="user icon"></i>Account</a>
-          <a class="item" href="/logout"><i class="sign out icon"></i>Logout</a>
+          <a class="item" href="/auth/account"><i class="user icon"></i>Account</a>
+          <a class="item" href="/auth/logout"><i class="sign out icon"></i>Logout</a>
         </div>
       </div>
-
+{% else %}
+      <div class="ui simple item">
+        <a href="/auth/login">Login/Signup</a>
+      </div>
+{% endif %}
     </div>
   </div>
 </header>
 
 <!-- 4em top margin is "enough" -->
 <main class="ui main container" style="margin-top: 6em; margin-bottom: 2em;">
+{% with messages = get_flashed_messages() %}
+  {% if messages %}
+    <div class="ui message">
+    {# Needs more javascript: <i class="close icon"></i> #}
+    <div class="header">Now Hear This...</div>
+    <ul class="list">
+    {% for message in messages %}
+      <li>{{ message }}
+    {% endfor %}
+    </ul>
+    </div>
+  {% endif %}
+{% endwith %}
 {% block fullbody %}
   <div class="ui container text">
     {% block body %}Nothing to see here.{% endblock %}
@@ -62,7 +80,7 @@
       <a class="item" href="https://guide.{{ config.FATCAT_DOMAIN }}/sources.html">Sources</a>
       <a class="item" href="{% if config.FATCAT_DOMAIN == "fatcat.wiki" %}https://stats.uptimerobot.com/GM9YNSrB0{% elif config.FATCAT_DOMAIN =="qa.fatcat.wiki" %}https://stats.uptimerobot.com/WQ8wAUREA{% else %}#{% endif %}">Status</a>
       <a class="item" href="https://guide.{{ config.FATCAT_DOMAIN }}/bulk_exports.html">Bulk Exports</a>
-      <a class="item" href="https://github.com/internetarchive/fatcat/">Source Code (<code>{{ config.GIT_REVISION.decode() }}</code>)</a>
+      <a class="item" href="https://github.com/internetarchive/fatcat/">Source Code (<code>{{ config.GIT_REVISION }}</code>)</a>
     </div>
   </div>
 </footer>
diff --git a/python/fatcat_web/templates/editor_changelog.html b/python/fatcat_web/templates/editor_changelog.html
index 79127312..785c19bd 100644
--- a/python/fatcat_web/templates/editor_changelog.html
+++ b/python/fatcat_web/templates/editor_changelog.html
@@ -3,8 +3,8 @@
 
 <h1 class="ui header">Editor Changelog: {{ editor.username }}
 <div class="sub header">
-  <a href="/editor/{{editor.id}}">
-    <code>editor {{ editor.id }}</code>
+  <a href="/editor/{{editor.editor_id}}">
+    <code>editor {{ editor.editor_id }}</code>
   </a>
 </div>
 </h1>
diff --git a/python/fatcat_web/templates/editor_view.html b/python/fatcat_web/templates/editor_view.html
index c9b61f5d..eef4f040 100644
--- a/python/fatcat_web/templates/editor_view.html
+++ b/python/fatcat_web/templates/editor_view.html
@@ -3,10 +3,10 @@
 
 <h1 class="ui header">{{ editor.username }}
 <div class="sub header">
-  <code>editor {{ editor.id }}</code>
+  <code>editor {{ editor.editor_id }}</code>
 </div>
 </h1>
 
-<p><b><a href="/editor/{{ editor.id }}/changelog">View editor's changelog</a></b>
+<p><b><a href="/editor/{{ editor.editor_id }}/changelog">View editor's changelog</a></b>
 
 {% endblock %}
diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py
new file mode 100644
index 00000000..cbe519b0
--- /dev/null
+++ b/python/fatcat_web/web_config.py
@@ -0,0 +1,64 @@
+
+"""
+Default configuration for fatcat web interface (Flask application).
+
+In production, we currently reconfigure these values using environment
+variables, not by (eg) deploying a variant copy of this file.
+
+This config is *only* for the web interface, *not* for any of the workers or
+import scripts.
+"""
+
+import os
+import raven
+import subprocess
+
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+class Config(object):
+    GIT_REVISION = subprocess.check_output(["git", "describe", "--always"]).strip().decode('utf-8')
+
+    # This is, effectively, the QA/PROD flag
+    FATCAT_DOMAIN = os.environ.get("FATCAT_DOMAIN", default="qa.fatcat.wiki")
+    FATCAT_API_AUTH_TOKEN = os.environ.get("FATCAT_API_AUTH_TOKEN", default=None)
+    FATCAT_API_HOST = os.environ.get("FATCAT_API_HOST", default="https://{}/v0".format(FATCAT_DOMAIN))
+
+    # can set this to https://search.fatcat.wiki for some experimentation
+    ELASTICSEARCH_BACKEND = os.environ.get("ELASTICSEARCH_BACKEND", default="http://localhost:9200")
+    ELASTICSEARCH_INDEX = os.environ.get("ELASTICSEARCH_INDEX", default="fatcat")
+
+    # for flask things, like session cookies
+    FLASK_SECRET_KEY = os.environ.get("FLASK_SECRET_KEY", default=None)
+    SECRET_KEY = FLASK_SECRET_KEY
+
+    GITLAB_CLIENT_ID = os.environ.get("GITLAB_CLIENT_ID", default=None)
+    GITLAB_CLIENT_SECRET = os.environ.get("GITLAB_CLIENT_SECRET", default=None)
+
+    IA_XAUTH_URI = "https://archive.org/services/xauthn/"
+    IA_XAUTH_CLIENT_ID = os.environ.get("IA_XAUTH_CLIENT_ID", default=None)
+    IA_XAUTH_CLIENT_SECRET = os.environ.get("IA_XAUTH_CLIENT_SECRET", default=None)
+
+    # protect cookies (which include API tokens)
+    SESSION_COOKIE_HTTPONLY = True
+    SESSION_COOKIE_SECURE = True
+    SESSION_COOKIE_SAMESITE = 'Lax'
+    PERMANENT_SESSION_LIFETIME = 2678400 # 31 days, in seconds
+
+    try:
+        GIT_RELEASE = raven.fetch_git_sha('..')
+    except Exception as e:
+        print("WARNING: couldn't set sentry git release automatically: " + str(e))
+        GIT_RELEASE = None
+
+    SENTRY_CONFIG = {
+        #'include_paths': ['fatcat_web', 'fatcat_client', 'fatcat_tools'],
+        'enable-threads': True, # for uWSGI
+        'release': GIT_RELEASE,
+        'tags': {
+            'fatcat_domain': FATCAT_DOMAIN,
+        },
+    }
+
+    # "Even more verbose" debug options
+    #SQLALCHEMY_ECHO = True
+    #DEBUG = True