diff options
31 files changed, 496 insertions, 93 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 9397c4ac..9a276ad2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ aliases: command: | sudo apt-get update sudo apt-get install -y sox libtag1v5 libmagic1 libffi6 ffmpeg postgresql-client-11 rsync - sudo apt-get install -y cmake build-essential git wget make libboost-all-dev + sudo apt-get install -y cmake build-essential git wget make libboost-all-dev rustc sudo apt-get install -y libsox-dev libsox-fmt-all libtag1-dev libmagic-dev libffi-dev libgd-dev libmad0-dev libsndfile1-dev libid3tag0-dev libmediainfo-dev - &install_audiowaveform @@ -41,9 +41,7 @@ aliases: - run: python -V | tee /tmp/.python-version - restore_cache: keys: - - v1-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }} - - v1-dependencies-{{ checksum "/tmp/.python-version" }}- - - v1-dependencies- + - v2-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }} - run: python3 -m venv venv - run: command: | @@ -53,7 +51,7 @@ aliases: pip install flake8 touch front/dist/index.html - save_cache: - key: v1-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }} + key: v2-dependencies-{{ checksum "/tmp/.python-version" }}-{{ checksum "api/requirements.txt" }} paths: - ./venv - *persist_to_workspace @@ -225,7 +223,13 @@ jobs: install-python3.8: <<: *defaults docker: - - image: circleci/python:3.8-rc-buster-node + - image: circleci/python:3.8-buster-node + <<: *install_python_dependencies + + install-python3.9: + <<: *defaults + docker: + - image: circleci/python:3.9-rc-buster-node <<: *install_python_dependencies test-python3.6: @@ -251,7 +255,17 @@ jobs: test-python3.8: <<: *defaults docker: - - image: circleci/python:3.8-rc-buster-node + - image: circleci/python:3.8-buster-node + - image: circleci/postgres:11-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_DB: reel2bits_test + <<: *test_steps + + test-python3.9: + <<: *defaults + docker: + - image: circleci/python:3.9-rc-buster-node - image: circleci/postgres:11-alpine environment: POSTGRES_USER: postgres @@ -313,32 +327,38 @@ workflows: - install-python3.7: requires: - install - - install-python3.6 + - install-python3.8: + requires: + - install + - install-python3.9: + requires: + - install - test-python3.6: requires: - install-python3.6 - test-python3.7: requires: - install-python3.7 + - test-python3.8: + requires: + - install-python3.8 + - test-python3.9: + requires: + - install-python3.9 - front-lint-lts: requires: - install - front-test-lts: requires: - - install - front-lint-lts - front-lint: requires: - install - - front-lint-lts - front-build: requires: - - install - - front-lint-lts - front-lint - front-sync: requires: - - install - front-lint-lts - front-build filters: @@ -34,3 +34,4 @@ front/locales/en_US/LC_MESSAGES/app.po .env /data /api/celerybeat-schedule +/venv diff --git a/.tmuxinator.yml b/.tmuxinator.yml index 81b05600..6a51b4a7 100644 --- a/.tmuxinator.yml +++ b/.tmuxinator.yml @@ -9,13 +9,15 @@ windows: - export AUTHLIB_INSECURE_TRANSPORT=1 - export APP_SETTINGS='config.development_secret.Config' - cd api + - source ../venv/bin/activate - flask run - workers: - export FLASK_ENV=development - export AUTHLIB_INSECURE_TRANSPORT=1 - export APP_SETTINGS='config.development_secret.Config' - cd api - - celery worker -A tasks.celery --loglevel=error + - source ../venv/bin/activate + - celery -A tasks.celery worker --loglevel=error - frontend: - cd front - yarn dev diff --git a/.vscode/settings.json b/.vscode/settings.json index e8f3a966..d2308c6f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -37,6 +37,6 @@ "vetur.format.defaultFormatter.js": "vscode-typescript", "vetur.format.defaultFormatter.html": "js-beautify-html", "javascript.format.insertSpaceBeforeFunctionParenthesis": true, - "python.pythonPath": "/usr/bin/python3.6", + "python.pythonPath": "venv/bin/python", "restructuredtext.confPath": "${workspaceFolder}/docs" }
\ No newline at end of file diff --git a/api/activitypub/backend.py b/api/activitypub/backend.py index 599273ce..78a623cc 100644 --- a/api/activitypub/backend.py +++ b/api/activitypub/backend.py @@ -90,7 +90,7 @@ class Reel2BitsBackend(ap.Backend): # Parse the activity ap_activity = ap.parse_activity(activity.payload) if not ap_activity: - current_app.logger.error(f"cannot parse undo follower activity") + current_app.logger.error("cannot parse undo follower activity") return actor = ap_activity.get_actor() @@ -159,7 +159,7 @@ class Reel2BitsBackend(ap.Backend): # FIXME TODO: check if it still works with unknown remote actor if not actor: - current_app.logger.debug(f"cannot find actor") + current_app.logger.debug("cannot find actor") actor = Actor.query.filter(Actor.url == ap_actor.id).first() if not actor: current_app.logger.debug(f"actor {ap_actor.id} not found") @@ -240,7 +240,7 @@ class Reel2BitsBackend(ap.Backend): current_app.logger.debug(f"fetch_iri: local activity {activity!r}") return activity.payload - current_app.logger.debug(f"fetch_iri: cannot find locally, fetching remote") + current_app.logger.debug("fetch_iri: cannot find locally, fetching remote") return super().fetch_iri(iri) def fetch_iri(self, iri: str) -> ap.ObjectType: diff --git a/api/activitypub/utils.py b/api/activitypub/utils.py index a44cbae8..3eb6030e 100644 --- a/api/activitypub/utils.py +++ b/api/activitypub/utils.py @@ -38,7 +38,7 @@ def full_url(path): def embed_collection(total_items, first_page_id): """Helper creating a root OrderedCollection - with a link to the first page.""" + with a link to the first page.""" return { "type": ap.ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, @@ -78,7 +78,7 @@ def activity_from_doc(item: Dict[str, Any], embed: bool = False) -> Dict[str, An def clean_activity(activity: ObjectType) -> Dict[str, Any]: """Clean the activity before rendering it. - - Remove the hidden bco and bcc field + - Remove the hidden bco and bcc field """ for field in ["bto", "bcc", "source"]: if field in activity: @@ -3,7 +3,7 @@ import logging import os import subprocess from logging.handlers import RotatingFileHandler -from flask_babelex import gettext, Babel +from flask_babel import gettext, Babel from flask import Flask, render_template, g, send_from_directory, jsonify, safe_join, request, flash, Response from flask_bootstrap import Bootstrap from flask_mail import Mail diff --git a/api/app_oauth.py b/api/app_oauth.py index 171f833b..98616f47 100644 --- a/api/app_oauth.py +++ b/api/app_oauth.py @@ -1,5 +1,5 @@ -from authlib.flask.oauth2 import AuthorizationServer, ResourceProtector -from authlib.flask.oauth2.sqla import ( +from authlib.integrations.flask_oauth2 import AuthorizationServer, ResourceProtector +from authlib.integrations.sqla_oauth2 import ( create_query_client_func, create_save_token_func, create_revocation_endpoint, diff --git a/api/authlib_sqla.py b/api/authlib_sqla.py new file mode 100644 index 00000000..4b3300f2 --- /dev/null +++ b/api/authlib_sqla.py @@ -0,0 +1,335 @@ +import time +import json +from sqlalchemy import Column, String, Boolean, Text, Integer +from sqlalchemy.ext.hybrid import hybrid_property +from authlib.oauth2.rfc6749 import ( + ClientMixin, + TokenMixin, + AuthorizationCodeMixin, +) +from authlib.oauth2.rfc6749.util import scope_to_list, list_to_scope +from authlib.oidc.core import AuthorizationCodeMixin as OIDCCodeMixin + + +class OAuth2ClientMixin(ClientMixin): + client_id = Column(String(48), index=True) + client_secret = Column(String(120)) + issued_at = Column(Integer, nullable=False, default=lambda: int(time.time())) + expires_at = Column(Integer, nullable=False, default=0) + + redirect_uri = Column(Text) + token_endpoint_auth_method = Column(String(48), default="client_secret_basic") + grant_type = Column(Text, nullable=False, default="") + response_type = Column(Text, nullable=False, default="") + scope = Column(Text, nullable=False, default="") + + client_name = Column(String(100)) + client_uri = Column(Text) + logo_uri = Column(Text) + contact = Column(Text) + tos_uri = Column(Text) + policy_uri = Column(Text) + jwks_uri = Column(Text) + jwks_text = Column(Text) + i18n_metadata = Column(Text) + + software_id = Column(String(36)) + software_version = Column(String(48)) + + def __repr__(self): + return "<Client: {}>".format(self.client_id) + + @hybrid_property + def redirect_uris(self): + if self.redirect_uri: + return self.redirect_uri.splitlines() + return [] + + @redirect_uris.setter + def redirect_uris(self, value): + self.redirect_uri = "\n".join(value) + + @hybrid_property + def grant_types(self): + if self.grant_type: + return self.grant_type.splitlines() + return [] + + @grant_types.setter + def grant_types(self, value): + self.grant_type = "\n".join(value) + + @hybrid_property + def response_types(self): + if self.response_type: + return self.response_type.splitlines() + return [] + + @response_types.setter + def response_types(self, value): + self.response_type = "\n".join(value) + + @hybrid_property + def contacts(self): + if self.contact: + return json.loads(self.contact) + return [] + + @contacts.setter + def contacts(self, value): + self.contact = json.dumps(value) + + @hybrid_property + def jwks(self): + if self.jwks_text: + return json.loads(self.jwks_text) + return None + + @jwks.setter + def jwks(self, value): + self.jwks_text = json.dumps(value) + + @hybrid_property + def client_metadata(self): + """Implementation for Client Metadata in OAuth 2.0 Dynamic Client + Registration Protocol via `Section 2`_. + + .. _`Section 2`: https://tools.ietf.org/html/rfc7591#section-2 + """ + keys = [ + "redirect_uris", + "token_endpoint_auth_method", + "grant_types", + "response_types", + "client_name", + "client_uri", + "logo_uri", + "scope", + "contacts", + "tos_uri", + "policy_uri", + "jwks_uri", + "jwks", + ] + metadata = {k: getattr(self, k) for k in keys} + if self.i18n_metadata: + metadata.update(json.loads(self.i18n_metadata)) + return metadata + + @client_metadata.setter + def client_metadata(self, value): + i18n_metadata = {} + for k in value: + if hasattr(self, k): + setattr(self, k, value[k]) + elif "#" in k: + i18n_metadata[k] = value[k] + + self.i18n_metadata = json.dumps(i18n_metadata) + + @property + def client_info(self): + """Implementation for Client Info in OAuth 2.0 Dynamic Client + Registration Protocol via `Section 3.2.1`_. + + .. _`Section 3.2.1`: https://tools.ietf.org/html/rfc7591#section-3.2.1 + """ + return dict( + client_id=self.client_id, + client_secret=self.client_secret, + client_id_issued_at=self.issued_at, + client_secret_expires_at=self.expires_at, + ) + + def get_client_id(self): + return self.client_id + + def get_default_redirect_uri(self): + if self.redirect_uris: + return self.redirect_uris[0] + + def get_allowed_scope(self, scope): + if not scope: + return "" + allowed = set(self.scope.split()) + scopes = scope_to_list(scope) + return list_to_scope([s for s in scopes if s in allowed]) + + def check_redirect_uri(self, redirect_uri): + return redirect_uri in self.redirect_uris + + def has_client_secret(self): + return bool(self.client_secret) + + def check_client_secret(self, client_secret): + return self.client_secret == client_secret + + def check_token_endpoint_auth_method(self, method): + return self.token_endpoint_auth_method == method + + def check_response_type(self, response_type): + if self.response_type: + return response_type in self.response_types + return False + + def check_grant_type(self, grant_type): + if self.grant_type: + return grant_type in self.grant_types + return False + + +class OAuth2AuthorizationCodeMixin(AuthorizationCodeMixin): + code = Column(String(120), unique=True, nullable=False) + client_id = Column(String(48)) + redirect_uri = Column(Text, default="") + response_type = Column(Text, default="") + scope = Column(Text, default="") + auth_time = Column(Integer, nullable=False, default=lambda: int(time.time())) + + def is_expired(self): + return self.auth_time + 300 < time.time() + + def get_redirect_uri(self): + return self.redirect_uri + + def get_scope(self): + return self.scope + + def get_auth_time(self): + return self.auth_time + + +class OIDCAuthorizationCodeMixin(OAuth2AuthorizationCodeMixin, OIDCCodeMixin): + nonce = Column(Text) + + def get_nonce(self): + return self.nonce + + +class OAuth2TokenMixin(TokenMixin): + client_id = Column(String(48)) + token_type = Column(String(40)) + access_token = Column(String(255), unique=True, nullable=False) + refresh_token = Column(String(255), index=True) + scope = Column(Text, default="") + revoked = Column(Boolean, default=False) + issued_at = Column(Integer, nullable=False, default=lambda: int(time.time())) + expires_in = Column(Integer, nullable=False, default=0) + + def get_client_id(self): + return self.client_id + + def get_scope(self): + return self.scope + + def get_expires_in(self): + return self.expires_in + + def get_expires_at(self): + return self.issued_at + self.expires_in + + +def create_query_client_func(session, client_model): + """Create an ``query_client`` function that can be used in authorization + server. + + :param session: SQLAlchemy session + :param client_model: Client model class + """ + + def query_client(client_id): + q = session.query(client_model) + return q.filter_by(client_id=client_id).first() + + return query_client + + +def create_save_token_func(session, token_model): + """Create an ``save_token`` function that can be used in authorization + server. + + :param session: SQLAlchemy session + :param token_model: Token model class + """ + + def save_token(token, request): + if request.user: + user_id = request.user.get_user_id() + else: + user_id = None + client = request.client + item = token_model(client_id=client.client_id, user_id=user_id, **token) + session.add(item) + session.commit() + + return save_token + + +def create_query_token_func(session, token_model): + """Create an ``query_token`` function for revocation, introspection + token endpoints. + + :param session: SQLAlchemy session + :param token_model: Token model class + """ + + def query_token(token, token_type_hint, client): + q = session.query(token_model) + q = q.filter_by(client_id=client.client_id, revoked=False) + if token_type_hint == "access_token": + return q.filter_by(access_token=token).first() + elif token_type_hint == "refresh_token": + return q.filter_by(refresh_token=token).first() + # without token_type_hint + item = q.filter_by(access_token=token).first() + if item: + return item + return q.filter_by(refresh_token=token).first() + + return query_token + + +def create_revocation_endpoint(session, token_model): + """Create a revocation endpoint class with SQLAlchemy session + and token model. + + :param session: SQLAlchemy session + :param token_model: Token model class + """ + from authlib.oauth2.rfc7009 import RevocationEndpoint + + query_token = create_query_token_func(session, token_model) + + class _RevocationEndpoint(RevocationEndpoint): + def query_token(self, token, token_type_hint, client): + return query_token(token, token_type_hint, client) + + def revoke_token(self, token): + token.revoked = True + session.add(token) + session.commit() + + return _RevocationEndpoint + + +def create_bearer_token_validator(session, token_model): + """Create an bearer token validator class with SQLAlchemy session + and token model. + + :param session: SQLAlchemy session + :param token_model: Token model class + """ + from authlib.oauth2.rfc6750 import BearerTokenValidator + + class _BearerTokenValidator(BearerTokenValidator): + def authenticate_token(self, token_string): + q = session.query(token_model) + return q.filter_by(access_token=token_string).first() + + def request_invalid(self, request): + return False + + def token_revoked(self, token): + return token.revoked + + return _BearerTokenValidator diff --git a/api/commands/system.py b/api/commands/system.py index 36bf24ed..9b55cdb1 100644 --- a/api/commands/system.py +++ b/api/commands/system.py @@ -40,7 +40,7 @@ def test_email(email): msg.html = render_template("email/test_email.html", instance=instance) try: mail.send(msg) - except: # noqa: E772 + except: # noqa: E722 print(f"Error sending mail: {sys.exc_info()[0]}") diff --git a/api/controllers/admin.py b/api/controllers/admin.py index 409a2f8c..239d4497 100644 --- a/api/controllers/admin.py +++ b/api/controllers/admin.py @@ -1,5 +1,5 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash, Response, json -from flask_babelex import gettext +from flask_babel import gettext from flask_security import login_required from forms import ConfigForm diff --git a/api/controllers/api/account.py b/api/controllers/api/account.py index 3230beef..b98af64a 100644 --- a/api/controllers/api/account.py +++ b/api/controllers/api/account.py @@ -1,6 +1,6 @@ from flask import Blueprint, jsonify, request, abort from app_oauth import require_oauth -from authlib.flask.oauth2 import current_token +from authlib.integrations.flask_oauth2 import current_token from models import UserLogging bp_api_account = Blueprint("bp_api_account", __name__) diff --git a/api/controllers/api/albums.py b/api/controllers/api/albums.py index 699baf4e..513581cd 100644 --- a/api/controllers/api/albums.py +++ b/api/controllers/api/albums.py @@ -1,6 +1,6 @@ from flask import Blueprint, jsonify, request, current_app from app_oauth import require_oauth -from authlib.flask.oauth2 import current_token +from authlib.integrations.flask_oauth2 import current_token from forms import AlbumForm from models import db, Album, User, Sound, SoundTag import json diff --git a/api/controllers/api/pleroma_admin.py b/api/controllers/api/pleroma_admin.py index 9fabdf1c..3f408d64 100644 --- a/api/controllers/api/pleroma_admin.py +++ b/api/controllers/api/pleroma_admin.py @@ -1,7 +1,7 @@ from flask import Blueprint, request, jsonify, abort from models import db, User from app_oauth import require_oauth -from authlib.flask.oauth2 import current_token +from authlib.integrations.flask_oauth2 import current_token bp_api_pleroma_admin = Blueprint("bp_api_pleroma_admin", __name__) diff --git a/api/controllers/api/reel2bits.py b/api/controllers/api/reel2bits.py index 77ab5f95..fddbd9b8 100644 --- a/api/controllers/api/reel2bits.py +++ b/api/controllers/api/reel2bits.py @@ -3,7 +3,7 @@ from models import db, User, PasswordResetToken, Sound, SoundTag, Actor, Followe from utils.various import add_user_log, generate_random_token, add_log from datas_helpers import to_json_account, to_json_relationship, default_genres, to_json_track from app_oauth import require_oauth -from authlib.flask.oauth2 import current_token +from authlib.integrations.flask_oauth2 import current_token from flask_security.utils import hash_password, verify_password from flask_mail import Message import smtplib diff --git a/api/controllers/api/tracks.py b/api/controllers/api/tracks.py index affeb990..492edb5e 100644 --- a/api/controllers/api/tracks.py +++ b/api/controllers/api/tracks.py @@ -1,6 +1,6 @@ from flask import Blueprint, request, jsonify, current_app from app_oauth import require_oauth -from authlib.flask.oauth2 import current_token +from authlib.integrations.flask_oauth2 import current_token from forms import SoundUploadForm from models import db, Sound, User, Album, UserLogging, SoundTag import json diff --git a/api/controllers/api/v1/accounts.py b/api/controllers/api/v1/accounts.py index eff33449..0ee3ad47 100644 --- a/api/controllers/api/v1/accounts.py +++ b/api/controllers/api/v1/accounts.py @@ -17,7 +17,7 @@ from models import ( from flask_security.utils import hash_password from flask_security import confirmable as FSConfi |