Merge pull request #378 from polyedre/fix-pipeline

31 files changed, 496 insertions, 93 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9397c4ac..9a276ad2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,7 +25,7 @@ aliases:
         command: |
           sudo apt-get update
           sudo apt-get install -y sox libtag1v5 libmagic1 libffi6 ffmpeg postgresql-client-11 rsync
-          sudo apt-get install -y cmake build-essential git wget make libboost-all-dev
+          sudo apt-get install -y cmake build-essential git wget make libboost-all-dev rustc
           sudo apt-get install -y libsox-dev libsox-fmt-all libtag1-dev libmagic-dev libffi-dev libgd-dev libmad0-dev libsndfile1-dev libid3tag0-dev libmediainfo-dev
 
   - &install_audiowaveform
@@ -41,9 +41,7 @@ aliases:
         - run: python -V | tee /tmp/.python-version
         - restore_cache:
             keys:
-              - v1-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }}
-              - v1-dependencies-{{ checksum "/tmp/.python-version" }}-
-              - v1-dependencies-
+              - v2-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }}
         - run: python3 -m venv venv
         - run:
             command: |
@@ -53,7 +51,7 @@ aliases:
               pip install flake8
               touch front/dist/index.html
         - save_cache:
-            key: v1-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }}
+            key: v2-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }}
             paths:
               - ./venv
         - *persist_to_workspace
@@ -225,7 +223,13 @@ jobs:
   install-python3.8:
     <<: *defaults
     docker:
-      - image: circleci/python:3.8-rc-buster-node
+      - image: circleci/python:3.8-buster-node
+    <<: *install_python_dependencies
+
+  install-python3.9:
+    <<: *defaults
+    docker:
+      - image: circleci/python:3.9-rc-buster-node
     <<: *install_python_dependencies
 
   test-python3.6:
@@ -251,7 +255,17 @@ jobs:
   test-python3.8:
     <<: *defaults
     docker:
-      - image: circleci/python:3.8-rc-buster-node
+      - image: circleci/python:3.8-buster-node
+      - image: circleci/postgres:11-alpine
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: reel2bits_test
+    <<: *test_steps
+
+  test-python3.9:
+    <<: *defaults
+    docker:
+      - image: circleci/python:3.9-rc-buster-node
       - image: circleci/postgres:11-alpine
         environment:
           POSTGRES_USER: postgres
@@ -313,32 +327,38 @@ workflows:
       - install-python3.7:
           requires:
             - install
-            - install-python3.6
+      - install-python3.8:
+          requires:
+            - install
+      - install-python3.9:
+          requires:
+            - install
       - test-python3.6:
           requires:
             - install-python3.6
       - test-python3.7:
           requires:
             - install-python3.7
+      - test-python3.8:
+          requires:
+            - install-python3.8
+      - test-python3.9:
+          requires:
+            - install-python3.9
       - front-lint-lts:
           requires:
             - install
       - front-test-lts:
           requires:
-            - install
             - front-lint-lts
       - front-lint:
           requires:
             - install
-            - front-lint-lts
       - front-build:
           requires:
-            - install
-            - front-lint-lts
             - front-lint
       - front-sync:
           requires:
-            - install
             - front-lint-lts
             - front-build
           filters:
diff --git a/.gitignore b/.gitignore
index 114d4611..469a2c64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,4 @@ front/locales/en_US/LC_MESSAGES/app.po
 .env
 /data
 /api/celerybeat-schedule
+/venv
diff --git a/.tmuxinator.yml b/.tmuxinator.yml
index 81b05600..6a51b4a7 100644
--- a/.tmuxinator.yml
+++ b/.tmuxinator.yml
@@ -9,13 +9,15 @@ windows:
       - export AUTHLIB_INSECURE_TRANSPORT=1
       - export APP_SETTINGS='config.development_secret.Config'
       - cd api
+      - source ../venv/bin/activate
       - flask run
   - workers:
       - export FLASK_ENV=development
       - export AUTHLIB_INSECURE_TRANSPORT=1
       - export APP_SETTINGS='config.development_secret.Config'
       - cd api
-      - celery worker -A tasks.celery --loglevel=error
+      - source ../venv/bin/activate
+      - celery -A tasks.celery worker --loglevel=error
   - frontend:
       - cd front
       - yarn dev
diff --git a/.vscode/settings.json b/.vscode/settings.json
index e8f3a966..d2308c6f 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -37,6 +37,6 @@
      "vetur.format.defaultFormatter.js": "vscode-typescript",
      "vetur.format.defaultFormatter.html": "js-beautify-html",
     "javascript.format.insertSpaceBeforeFunctionParenthesis": true,
-    "python.pythonPath": "/usr/bin/python3.6",
+    "python.pythonPath": "venv/bin/python",
     "restructuredtext.confPath": "${workspaceFolder}/docs"
 }
\ No newline at end of file
diff --git a/api/activitypub/backend.py b/api/activitypub/backend.py
index 599273ce..78a623cc 100644
--- a/api/activitypub/backend.py
+++ b/api/activitypub/backend.py
@@ -90,7 +90,7 @@ class Reel2BitsBackend(ap.Backend):
         # Parse the activity
         ap_activity = ap.parse_activity(activity.payload)
         if not ap_activity:
-            current_app.logger.error(f"cannot parse undo follower activity")
+            current_app.logger.error("cannot parse undo follower activity")
             return
 
         actor = ap_activity.get_actor()
@@ -159,7 +159,7 @@ class Reel2BitsBackend(ap.Backend):
 
         # FIXME TODO: check if it still works with unknown remote actor
         if not actor:
-            current_app.logger.debug(f"cannot find actor")
+            current_app.logger.debug("cannot find actor")
             actor = Actor.query.filter(Actor.url == ap_actor.id).first()
             if not actor:
                 current_app.logger.debug(f"actor {ap_actor.id} not found")
@@ -240,7 +240,7 @@ class Reel2BitsBackend(ap.Backend):
                 current_app.logger.debug(f"fetch_iri: local activity {activity!r}")
                 return activity.payload
 
-        current_app.logger.debug(f"fetch_iri: cannot find locally, fetching remote")
+        current_app.logger.debug("fetch_iri: cannot find locally, fetching remote")
         return super().fetch_iri(iri)
 
     def fetch_iri(self, iri: str) -> ap.ObjectType:
diff --git a/api/activitypub/utils.py b/api/activitypub/utils.py
index a44cbae8..3eb6030e 100644
--- a/api/activitypub/utils.py
+++ b/api/activitypub/utils.py
@@ -38,7 +38,7 @@ def full_url(path):
 
 def embed_collection(total_items, first_page_id):
     """Helper creating a root OrderedCollection
-     with a link to the first page."""
+    with a link to the first page."""
     return {
         "type": ap.ActivityType.ORDERED_COLLECTION.value,
         "totalItems": total_items,
@@ -78,7 +78,7 @@ def activity_from_doc(item: Dict[str, Any], embed: bool = False) -> Dict[str, An
 
 def clean_activity(activity: ObjectType) -> Dict[str, Any]:
     """Clean the activity before rendering it.
-     - Remove the hidden bco and bcc field
+    - Remove the hidden bco and bcc field
     """
     for field in ["bto", "bcc", "source"]:
         if field in activity:
diff --git a/api/app.py b/api/app.py
index 8dfae508..6b58cfb4 100644
--- a/api/app.py
+++ b/api/app.py
@@ -3,7 +3,7 @@ import logging
 import os
 import subprocess
 from logging.handlers import RotatingFileHandler
-from flask_babelex import gettext, Babel
+from flask_babel import gettext, Babel
 from flask import Flask, render_template, g, send_from_directory, jsonify, safe_join, request, flash, Response
 from flask_bootstrap import Bootstrap
 from flask_mail import Mail
diff --git a/api/app_oauth.py b/api/app_oauth.py
index 171f833b..98616f47 100644
--- a/api/app_oauth.py
+++ b/api/app_oauth.py
@@ -1,5 +1,5 @@
-from authlib.flask.oauth2 import AuthorizationServer, ResourceProtector
-from authlib.flask.oauth2.sqla import (
+from authlib.integrations.flask_oauth2 import AuthorizationServer, ResourceProtector
+from authlib.integrations.sqla_oauth2 import (
     create_query_client_func,
     create_save_token_func,
     create_revocation_endpoint,
diff --git a/api/authlib_sqla.py b/api/authlib_sqla.py
new file mode 100644
index 00000000..4b3300f2
--- /dev/null
+++ b/api/authlib_sqla.py
@@ -0,0 +1,335 @@
+import time
+import json
+from sqlalchemy import Column, String, Boolean, Text, Integer
+from sqlalchemy.ext.hybrid import hybrid_property
+from authlib.oauth2.rfc6749 import (
+    ClientMixin,
+    TokenMixin,
+    AuthorizationCodeMixin,
+)
+from authlib.oauth2.rfc6749.util import scope_to_list, list_to_scope
+from authlib.oidc.core import AuthorizationCodeMixin as OIDCCodeMixin
+
+
+class OAuth2ClientMixin(ClientMixin):
+    client_id = Column(String(48), index=True)
+    client_secret = Column(String(120))
+    issued_at = Column(Integer, nullable=False, default=lambda: int(time.time()))
+    expires_at = Column(Integer, nullable=False, default=0)
+
+    redirect_uri = Column(Text)
+    token_endpoint_auth_method = Column(String(48), default="client_secret_basic")
+    grant_type = Column(Text, nullable=False, default="")
+    response_type = Column(Text, nullable=False, default="")
+    scope = Column(Text, nullable=False, default="")
+
+    client_name = Column(String(100))
+    client_uri = Column(Text)
+    logo_uri = Column(Text)
+    contact = Column(Text)
+    tos_uri = Column(Text)
+    policy_uri = Column(Text)
+    jwks_uri = Column(Text)
+    jwks_text = Column(Text)
+    i18n_metadata = Column(Text)
+
+    software_id = Column(String(36))
+    software_version = Column(String(48))
+
+    def __repr__(self):
+        return "<Client: {}>".format(self.client_id)
+
+    @hybrid_property
+    def redirect_uris(self):
+        if self.redirect_uri:
+            return self.redirect_uri.splitlines()
+        return []
+
+    @redirect_uris.setter
+    def redirect_uris(self, value):
+        self.redirect_uri = "\n".join(value)
+
+    @hybrid_property
+    def grant_types(self):
+        if self.grant_type:
+            return self.grant_type.splitlines()
+        return []
+
+    @grant_types.setter
+    def grant_types(self, value):
+        self.grant_type = "\n".join(value)
+
+    @hybrid_property
+    def response_types(self):
+        if self.response_type:
+            return self.response_type.splitlines()
+        return []
+
+    @response_types.setter
+    def response_types(self, value):
+        self.response_type = "\n".join(value)
+
+    @hybrid_property
+    def contacts(self):
+        if self.contact:
+            return json.loads(self.contact)
+        return []
+
+    @contacts.setter
+    def contacts(self, value):
+        self.contact = json.dumps(value)
+
+    @hybrid_property
+    def jwks(self):
+        if self.jwks_text:
+            return json.loads(self.jwks_text)
+        return None
+
+    @jwks.setter
+    def jwks(self, value):
+        self.jwks_text = json.dumps(value)
+
+    @hybrid_property
+    def client_metadata(self):
+        """Implementation for Client Metadata in OAuth 2.0 Dynamic Client
+        Registration Protocol via `Section 2`_.
+
+        .. _`Section 2`: https://tools.ietf.org/html/rfc7591#section-2
+        """
+        keys = [
+            "redirect_uris",
+            "token_endpoint_auth_method",
+            "grant_types",
+            "response_types",
+            "client_name",
+            "client_uri",
+            "logo_uri",
+            "scope",
+            "contacts",
+            "tos_uri",
+            "policy_uri",
+            "jwks_uri",
+            "jwks",
+        ]
+        metadata = {k: getattr(self, k) for k in keys}
+        if self.i18n_metadata:
+            metadata.update(json.loads(self.i18n_metadata))
+        return metadata
+
+    @client_metadata.setter
+    def client_metadata(self, value):
+        i18n_metadata = {}
+        for k in value:
+            if hasattr(self, k):
+                setattr(self, k, value[k])
+            elif "#" in k:
+                i18n_metadata[k] = value[k]
+
+        self.i18n_metadata = json.dumps(i18n_metadata)
+
+    @property
+    def client_info(self):
+        """Implementation for Client Info in OAuth 2.0 Dynamic Client
+        Registration Protocol via `Section 3.2.1`_.
+
+        .. _`Section 3.2.1`: https://tools.ietf.org/html/rfc7591#section-3.2.1
+        """
+        return dict(
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            client_id_issued_at=self.issued_at,
+            client_secret_expires_at=self.expires_at,
+        )
+
+    def get_client_id(self):
+        return self.client_id
+
+    def get_default_redirect_uri(self):
+        if self.redirect_uris:
+            return self.redirect_uris[0]
+
+    def get_allowed_scope(self, scope):
+        if not scope:
+            return ""
+        allowed = set(self.scope.split())
+        scopes = scope_to_list(scope)
+        return list_to_scope([s for s in scopes if s in allowed])
+
+    def check_redirect_uri(self, redirect_uri):
+        return redirect_uri in self.redirect_uris
+
+    def has_client_secret(self):
+        return bool(self.client_secret)
+
+    def check_client_secret(self, client_secret):
+        return self.client_secret == client_secret
+
+    def check_token_endpoint_auth_method(self, method):
+        return self.token_endpoint_auth_method == method
+
+    def check_response_type(self, response_type):
+        if self.response_type:
+            return response_type in self.response_types
+        return False
+
+    def check_grant_type(self, grant_type):
+        if self.grant_type:
+            return grant_type in self.grant_types
+        return False
+
+
+class OAuth2AuthorizationCodeMixin(AuthorizationCodeMixin):
+    code = Column(String(120), unique=True, nullable=False)
+    client_id = Column(String(48))
+    redirect_uri = Column(Text, default="")
+    response_type = Column(Text, default="")
+    scope = Column(Text, default="")
+    auth_time = Column(Integer, nullable=False, default=lambda: int(time.time()))
+
+    def is_expired(self):
+        return self.auth_time + 300 < time.time()
+
+    def get_redirect_uri(self):
+        return self.redirect_uri
+
+    def get_scope(self):
+        return self.scope
+
+    def get_auth_time(self):
+        return self.auth_time
+
+
+class OIDCAuthorizationCodeMixin(OAuth2AuthorizationCodeMixin, OIDCCodeMixin):
+    nonce = Column(Text)
+
+    def get_nonce(self):
+        return self.nonce
+
+
+class OAuth2TokenMixin(TokenMixin):
+    client_id = Column(String(48))
+    token_type = Column(String(40))
+    access_token = Column(String(255), unique=True, nullable=False)
+    refresh_token = Column(String(255), index=True)
+    scope = Column(Text, default="")
+    revoked = Column(Boolean, default=False)
+    issued_at = Column(Integer, nullable=False, default=lambda: int(time.time()))
+    expires_in = Column(Integer, nullable=False, default=0)
+
+    def get_client_id(self):
+        return self.client_id
+
+    def get_scope(self):
+        return self.scope
+
+    def get_expires_in(self):
+        return self.expires_in
+
+    def get_expires_at(self):
+        return self.issued_at + self.expires_in
+
+
+def create_query_client_func(session, client_model):
+    """Create an ``query_client`` function that can be used in authorization
+    server.
+
+    :param session: SQLAlchemy session
+    :param client_model: Client model class
+    """
+
+    def query_client(client_id):
+        q = session.query(client_model)
+        return q.filter_by(client_id=client_id).first()
+
+    return query_client
+
+
+def create_save_token_func(session, token_model):
+    """Create an ``save_token`` function that can be used in authorization
+    server.
+
+    :param session: SQLAlchemy session
+    :param token_model: Token model class
+    """
+
+    def save_token(token, request):
+        if request.user:
+            user_id = request.user.get_user_id()
+        else:
+            user_id = None
+        client = request.client
+        item = token_model(client_id=client.client_id, user_id=user_id, **token)
+        session.add(item)
+        session.commit()
+
+    return save_token
+
+
+def create_query_token_func(session, token_model):
+    """Create an ``query_token`` function for revocation, introspection
+    token endpoints.
+
+    :param session: SQLAlchemy session
+    :param token_model: Token model class
+    """
+
+    def query_token(token, token_type_hint, client):
+        q = session.query(token_model)
+        q = q.filter_by(client_id=client.client_id, revoked=False)
+        if token_type_hint == "access_token":
+            return q.filter_by(access_token=token).first()
+        elif token_type_hint == "refresh_token":
+            return q.filter_by(refresh_token=token).first()
+        # without token_type_hint
+        item = q.filter_by(access_token=token).first()
+        if item:
+            return item
+        return q.filter_by(refresh_token=token).first()
+
+    return query_token
+
+
+def create_revocation_endpoint(session, token_model):
+    """Create a revocation endpoint class with SQLAlchemy session
+    and token model.
+
+    :param session: SQLAlchemy session
+    :param token_model: Token model class
+    """
+    from authlib.oauth2.rfc7009 import RevocationEndpoint
+
+    query_token = create_query_token_func(session, token_model)
+
+    class _RevocationEndpoint(RevocationEndpoint):
+        def query_token(self, token, token_type_hint, client):
+            return query_token(token, token_type_hint, client)
+
+        def revoke_token(self, token):
+            token.revoked = True
+            session.add(token)
+            session.commit()
+
+    return _RevocationEndpoint
+
+
+def create_bearer_token_validator(session, token_model):
+    """Create an bearer token validator class with SQLAlchemy session
+    and token model.
+
+    :param session: SQLAlchemy session
+    :param token_model: Token model class
+    """
+    from authlib.oauth2.rfc6750 import BearerTokenValidator
+
+    class _BearerTokenValidator(BearerTokenValidator):
+        def authenticate_token(self, token_string):
+            q = session.query(token_model)
+            return q.filter_by(access_token=token_string).first()
+
+        def request_invalid(self, request):
+            return False
+
+        def token_revoked(self, token):
+            return token.revoked
+
+    return _BearerTokenValidator
diff --git a/api/commands/system.py b/api/commands/system.py
index 36bf24ed..9b55cdb1 100644
--- a/api/commands/system.py
+++ b/api/commands/system.py
@@ -40,7 +40,7 @@ def test_email(email):
     msg.html = render_template("email/test_email.html", instance=instance)
     try:
         mail.send(msg)
-    except:  # noqa: E772
+    except:  # noqa: E722
         print(f"Error sending mail: {sys.exc_info()[0]}")
 
 
diff --git a/api/controllers/admin.py b/api/controllers/admin.py
index 409a2f8c..239d4497 100644
--- a/api/controllers/admin.py
+++ b/api/controllers/admin.py
@@ -1,5 +1,5 @@
 from flask import Blueprint, render_template, request, redirect, url_for, flash, Response, json
-from flask_babelex import gettext
+from flask_babel import gettext
 from flask_security import login_required
 
 from forms import ConfigForm
diff --git a/api/controllers/api/account.py b/api/controllers/api/account.py
index 3230beef..b98af64a 100644
--- a/api/controllers/api/account.py
+++ b/api/controllers/api/account.py
@@ -1,6 +1,6 @@
 from flask import Blueprint, jsonify, request, abort
 from app_oauth import require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 from models import UserLogging
 
 bp_api_account = Blueprint("bp_api_account", __name__)
diff --git a/api/controllers/api/albums.py b/api/controllers/api/albums.py
index 699baf4e..513581cd 100644
--- a/api/controllers/api/albums.py
+++ b/api/controllers/api/albums.py
@@ -1,6 +1,6 @@
 from flask import Blueprint, jsonify, request, current_app
 from app_oauth import require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 from forms import AlbumForm
 from models import db, Album, User, Sound, SoundTag
 import json
diff --git a/api/controllers/api/pleroma_admin.py b/api/controllers/api/pleroma_admin.py
index 9fabdf1c..3f408d64 100644
--- a/api/controllers/api/pleroma_admin.py
+++ b/api/controllers/api/pleroma_admin.py
@@ -1,7 +1,7 @@
 from flask import Blueprint, request, jsonify, abort
 from models import db, User
 from app_oauth import require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 
 bp_api_pleroma_admin = Blueprint("bp_api_pleroma_admin", __name__)
 
diff --git a/api/controllers/api/reel2bits.py b/api/controllers/api/reel2bits.py
index 77ab5f95..fddbd9b8 100644
--- a/api/controllers/api/reel2bits.py
+++ b/api/controllers/api/reel2bits.py
@@ -3,7 +3,7 @@ from models import db, User, PasswordResetToken, Sound, SoundTag, Actor, Followe
 from utils.various import add_user_log, generate_random_token, add_log
 from datas_helpers import to_json_account, to_json_relationship, default_genres, to_json_track
 from app_oauth import require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 from flask_security.utils import hash_password, verify_password
 from flask_mail import Message
 import smtplib
diff --git a/api/controllers/api/tracks.py b/api/controllers/api/tracks.py
index affeb990..492edb5e 100644
--- a/api/controllers/api/tracks.py
+++ b/api/controllers/api/tracks.py
@@ -1,6 +1,6 @@
 from flask import Blueprint, request, jsonify, current_app
 from app_oauth import require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 from forms import SoundUploadForm
 from models import db, Sound, User, Album, UserLogging, SoundTag
 import json
diff --git a/api/controllers/api/v1/accounts.py b/api/controllers/api/v1/accounts.py
index eff33449..0ee3ad47 100644
--- a/api/controllers/api/v1/accounts.py
+++ b/api/controllers/api/v1/accounts.py
@@ -17,7 +17,7 @@ from models import (
 from flask_security.utils import hash_password
 from flask_security import confirmable as FSConfirmable
 from app_oauth import authorization, require_oauth
-from authlib.flask.oauth2 import current_token
+from authlib.integrations.flask_oauth2 import current_token
 from datas_helpers import to_json_track, to_json_account, to_json_relationship
 from utils.various import forbidden_username, add_user_log, add_log, get_hashed_filename
 from tasks import send_update_profile, federate_delete_actor, post_to_outbox
@@ -42,6 +42,8 @@ avatars = UploadSet("avatars", Reel2bitsDefaults.avatar_extensions_allowed)