diff options
author | Danglewood <85772166+deeleeramone@users.noreply.github.com> | 2024-07-01 10:51:21 -0700 |
---|---|---|
committer | Danglewood <85772166+deeleeramone@users.noreply.github.com> | 2024-07-01 10:51:21 -0700 |
commit | a417725f76406427ebb45e986bb1cc87d06277d5 (patch) | |
tree | f67d724efd1886fcf434c94588c19d03e4efebb1 | |
parent | b2cbe4afaa3e1f1f260781a12058353b87f7b492 (diff) | |
parent | ce2816a5a31245442ce05c49e69138466e558e4b (diff) |
Merge branch 'feature/delay-import' of https://github.com/OpenBB-finance/OpenBBTerminal into feature/delay-import
-rw-r--r-- | build/pypi/openbb_platform/PUBLISH.md | 90 | ||||
-rw-r--r-- | build/pypi/openbb_platform/publish.py | 255 | ||||
-rw-r--r-- | openbb_platform/core/openbb_core/app/static/utils/linters.py | 8 |
3 files changed, 227 insertions, 126 deletions
diff --git a/build/pypi/openbb_platform/PUBLISH.md b/build/pypi/openbb_platform/PUBLISH.md index 04fb747b7ed..53bb251a47c 100644 --- a/build/pypi/openbb_platform/PUBLISH.md +++ b/build/pypi/openbb_platform/PUBLISH.md @@ -1,90 +1,60 @@ # Publishing to PyPI -## Pre-release procedure +## Pre-release > [!WARNING] -> The `release` branch is solely for the purpose of publishing the package(s)! Any last minute changes should be made in appropriate PRs and merged to the `develop` branch. Once the `release` branch is created, the `develop` branch should be frozen for further commits. +> The `release` branch is solely for the purpose of publishing the package(s)! Any last minute changes should be made in appropriate PRs and merged to the `develop` branch. Once the `release` branch is created, the `develop` branch should be frozen for further commits until this process finishes. > A `release` branch for a particular package should only cater to the changes for that package directory. For e.g. the `release/openbb-core-2.0.0` branch should only contain changes for the `openbb-core` package i.e. in the `openbb_platform/core` directory. -### Flow to display only files changes during release cycle in the `release/...` -> `main` PR - -1. Merge main into develop before release branch exists, solve any conflict -2. In the branch `release/โฆ` (before or after publishing) -3. `git merge main -X ours` (ignore the changes, we brought them in 1.) -4. `git commit -m "Merge branch 'main' into release/4.2.2"` (empty commit) +0. Merge `main` into `develop` and solve any conflict. You can do this by checking out `develop`, creating a branch `feature/merge-main-into-develop` and running `git merge main`. Merge this into `develop` if there are changes to commit. +1. Open a PR with the changes to be published in the format `release/<version>` (for e.g. `release/4.0.0` ). For a particular package use the format `release/<package>-<version>` (for e.g. `release/openbb-core-1.0.1`). +2. Bump `openbb` package version in `openbb_platform/pyproject.toml`. -### Pre-release checklist +> [!WARNING] +> The version must be incremented before running the tests in 4. and 5., since there are tests that rely of this to check for deprecated endpoints. -1. Open a PR with the changes to be published in the format `release/<version>` (for e.g. `release/4.0.0` ). For a particular package use the format `release/<package>-<version>` (for e.g. `release/openbb-core-1.0.1`). -2. Ensure all the CI workflows pass. -3. Ensure all unit tests pass: `pytest openbb_platform -m "not integration"` -4. Ensure all integration tests pass: `pytest openbb_platform -m integration` +3. Ensure all the CI workflows pass. +4. Ensure all unit tests pass: `pytest openbb_platform -m "not integration"` +5. Ensure all integration tests pass: `pytest openbb_platform -m integration` -## Release procedure +## Release > Ensure you have the appropriate credentials and permissions to publish to PyPI. 1. Run the following commands for publishing the packages to PyPI: - Consider using the `--dry-run` flag to check if everything is correct before publishing. - Also, it might be a good idea to run the script in batches to ensure that the packages are published correctly and the dependencies pick the correct versions. - - > For a single package release, the following steps are optional since the package can be bumped manually. - - 1. For the core package run: `python build/pypi/openbb_platform/publish.py --core` - 2. For the extension packages run: `python build/pypi/openbb_platform/publish.py --extensions` - 3. For the `openbb` package - **which requires manual publishing** - do the following - - 3.1. Bump the `openbb` package version and the extension versions on `openbb_platform/pyproject.toml` to the latest version. - - > [!TIP] - > Consider using the poetry plugin `up` for updating the extensions to the latest version: - > 1. `poetry self add poetry-plugin-up` - > 2. `poetry up --latest` - - > [!WARNING] - > Create a new environment before proceeding. - > Make sure that only required extensions are installed + > [!TIP] + > Consider using the `--dry-run` flag to check if everything is correct before publishing. - 3.2. Run `pip install -e .` from `openbb_platform` + > For a single package release (e.g. patching `openbb-core`), the following steps are not required since the package can be published manually. - 3.3. Re-build the static assets that are bundled with the package: `python -c "import openbb; openbb.build()"` - - Run `python -c "import openbb"` after building the static to check that no additional static is being built. - - Run any command to smoke test if the static assets are being built correctly. + 1. To publish `openbb-core` run: `python build/pypi/openbb_platform/publish.py --core` + 2. To publish **ALL** extensions run: `python build/pypi/openbb_platform/publish.py --extensions` + 3. To publish `openbb` (the main package) run: `python build/pypi/openbb_platform/publish.py --openbb` + > [!TIP] + > Note that, in order for packages to pick up the latest versions of dependencies, it is advised to clear the local cache of the dependencies: we can do that with `pip cache purge` and `poetry cache clear pypi --all`. Sometimes there might be some delay in the PyPI API, so it might be necessary to wait a few minutes for pip to pick the latest versions. - 3.4. Run unit tests to validate the existence of deprecated endpoints (or watch this through GitHub Actions) +2. Publish the CLI - 3.5. Run `poetry publish --build` from `openbb_platform` + 2.1. Bump `openbb` dependency on `cli/pyproject.toml` to the latest version - 3.6. Run `poetry lock` from `openbb_platform` + 2.2. Run `poetry publish --build` from `cli` - > [!TIP] - > Note that, in order for packages to pick up the latest versions of dependencies, it is advised to clear the local cache of the dependencies: - > - > We can do that with `pip cache purge` and `poetry cache clear pypi --all` - > - > Also, sometimes there might be some delay in the PyPI API, so it might be necessary to wait a few minutes before publishing the next package. +3. Regenerate assets for external use by running `python assets/scripts/generate_extension_data.py`. +4. Merge `release/<package>-<version>` to the `main` branch. +5. Run `Deploy to GitHub Pages` action in [openbb-docs](https://github.com/OpenBB-finance/openbb-docs/actions) to update the documentation website. Go to [docs.openbb.co](https://docs.openbb.co) to see the changes. -2. Merge the `release/<package>-<version>` branch to the `main` branch. -3. Run the `Deploy to GitHub Pages` action in [openbb-docs](https://github.com/OpenBB-finance/openbb-docs/actions). Go to the [docs](https://docs.openbb.co) website to see the changes. - -## Post-release procedure +## Post-release 1. Install the packages on Google Colaboratory via PyPi and test to check if everything is working as expected. 2. Install the packages in a new environment locally via PyPi and test to check if everything is working as expected. -3. Regenerate assets for external use by running `python assets/scripts/generate_extension_data.py` -4. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch. -5. Merge the `release/<package>-<version>` branch to the `develop` branch. -6. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly. +3. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch. +4. Merge the `release/<package>-<version>` branch to the `develop` branch. +5. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly. ### Generate the changelog -1. Run the changelog automation by using the "release_drafter" GA and passing the number of the previous Release. +1. Run the changelog automation by using the "release_drafter" GitHub action and input the number of the last release PR (not this one!). 2. Edit and make the changelog live on the repository. 3. Paste it in the platform-release-changelog Slack channel. - -### Publish the CLI - -1. Bump `openbb` dependency on `cli/pyproject.toml` to the latest version. -2. Run `poetry publish --build` from `cli` diff --git a/build/pypi/openbb_platform/publish.py b/build/pypi/openbb_platform/publish.py index 6d163af0860..ac6d48bf7ff 100644 --- a/build/pypi/openbb_platform/publish.py +++ b/build/pypi/openbb_platform/publish.py @@ -1,19 +1,23 @@ """Publish the OpenBB Platform to PyPi.""" import argparse -import subprocess +import logging import sys +from functools import partial from pathlib import Path +from subprocess import PIPE, run +from typing import Literal -PLATFORM_PATH = Path(__file__).parent.parent.parent.parent.resolve() / "openbb_platform" +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +console = logging.StreamHandler() +logger.addHandler(console) +formatter = logging.Formatter("[%(levelname)s] %(message)s") +console.setFormatter(formatter) -CORE_PACKAGES = ["core"] -EXTENSION_PACKAGES = ["extensions", "providers", "obbject_extensions"] - -CMD = [sys.executable, "-m", "poetry"] -EXTENSION_DEPENDENCIES_UPDATE_CMD = ["add", "openbb-core=latest", "--lock"] -VERSION_BUMP_CMD = ["version", "patch"] -PUBLISH_CMD = ["publish", "--build"] +DIR_PLATFORM = Path(__file__).parent.parent.parent.parent.resolve() / "openbb_platform" +DIR_CORE = ["core"] +DIR_EXTENSIONS = ["extensions", "providers", "obbject_extensions"] def parse_args(): @@ -27,84 +31,209 @@ def parse_args(): "-e", "--extensions", action="store_true", - help="Publish extension packages.", + help="Publish extension packages, such as openbb-equity or openbb-fmp.", dest="extensions", ) parser.add_argument( + "-o", + "--openbb", + action="store_true", + help="Publish openbb package.", + dest="openbb", + ) + parser.add_argument( "--dry-run", action="store_true", help="Run the commands without actually publishing.", default=False, dest="dry_run", ) - return parser.parse_args() - - -def update_extension_dependencies(path: Path): - """Update the extension dependencies""" - subprocess.run( - CMD + EXTENSION_DEPENDENCIES_UPDATE_CMD, # noqa: S603 - cwd=path.parent, - check=True, + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Run the tool in verbose mode.", + default=False, + dest="verbose", ) + parser.add_argument( + "-y", + "--yes", + action="store_true", + help="Answer input questions.", + default=False, + dest="yes", + ) + parser.add_argument( + "--semver", + help="Semantic version.", + default="patch", + choices=["patch", "minor", "major"], + dest="semver", + ) + return parser.parse_args() -def bump_version(path: Path): - """Bump the version of the package""" - print(f"\n> {path.parent.stem}") # noqa: T201 - subprocess.run(CMD + VERSION_BUMP_CMD, cwd=path.parent, check=True) # noqa: S603 - - -def publish(dry_run: bool = False, core: bool = False, extensions: bool = False): +def publish( + dry_run: bool = False, + core: bool = False, + extensions: bool = False, + openbb: bool = False, + verbose: bool = False, + semver: Literal["patch", "minor", "major"] = "patch", +): """Publish the Platform to PyPi with optional core or extensions.""" - package_paths = [] + package_directories = [] if core: - print("Working with core packages...") # noqa: T201 - package_paths.extend(CORE_PACKAGES) + package_directories.extend(DIR_CORE) if extensions: - print("Working with extensions...") # noqa: T201 - package_paths.extend(EXTENSION_PACKAGES) + package_directories.extend(DIR_EXTENSIONS) - for sub_path in package_paths: - is_extension = sub_path in EXTENSION_PACKAGES + partial_run = partial( + run, + check=True, + stdout=None if verbose else PIPE, + stderr=None if verbose else PIPE, + ) - for path in sorted(PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml")): + for _dir in package_directories: + is_extension = _dir in DIR_EXTENSIONS + paths = [ + p + for p in sorted(DIR_PLATFORM.rglob(f"{_dir}/**/pyproject.toml")) + if "devtools" not in str(p) + ] + total = len(paths) + logger.info("~~~ /%s ~~~", _dir) + for i, path in enumerate(paths): + logger.info( + "๐ (%s/%s) Publishing openbb-%s...", + i + 1, + total, + path.parent.stem.replace("_", "-"), + ) try: - # Update dependencies - if is_extension and "devtools" not in str(path): - update_extension_dependencies(path) - # Bump version - bump_version(path) - # Publish (if not dry run) + # Update openbb-core to latest in each pyproject.toml + if is_extension: + partial_run( + [ + sys.executable, + "-m", + "poetry", + "add", + "openbb-core=latest", + "--lock", + ], + cwd=path.parent, + ) + # Bump pyproject.toml version + partial_run( + [sys.executable, "-m", "poetry", "version", semver], + cwd=path.parent, + ) + # Publish (if not dry running) if not dry_run: - subprocess.run( - CMD + PUBLISH_CMD, # noqa: S603 + partial_run( + [ + sys.executable, + "-m", + "poetry", + "build", + "publish", + "--build", + ], cwd=path.parent, - check=True, # noqa: S603 ) + logger.info("โ
Success") except Exception as e: - print(f"Error publishing {path.parent}:\n{e}") # noqa: T201 + logger.error("โ Failed to publish %s:\n\n%s", path.parent.stem, e) + + if openbb: + STEPS = 7 + logger.info("~~~ /openbb ~~~") + logger.info("๐งฉ (1/%s) Installing poetry-plugin-up...", STEPS) + partial_run( + ["pip", "install", "poetry-plugin-up"], + cwd=DIR_PLATFORM, + ) + logger.info("โซ (2/%s) Updating openbb pyproject.toml...", STEPS) + partial_run( + [sys.executable, "-m", "poetry", "up", "--latest"], + cwd=DIR_PLATFORM, + ) + logger.info("๐ (3/%s) Writing openbb poetry.lock...", STEPS) + partial_run( + [sys.executable, "-m", "poetry", "up", "--latest"], + cwd=DIR_PLATFORM, + ) + logger.info("๐ (4/%s) Installing openbb from /%s...", STEPS, DIR_PLATFORM.stem) + partial_run( + ["pip", "install", "-U", "--editable", "."], + cwd=DIR_PLATFORM, + ) + logger.info("๐ง (5/%s) Building python interface...", STEPS) + result = run( + [sys.executable, "-c", "import openbb; openbb.build()"], # noqa: S603 + cwd=DIR_PLATFORM, + check=True, + capture_output=True, + text=True, + ) + if verbose: + logger.info("Captured result -> %s", result) + if result.stderr: + logger.error("โ stderr is not empty!") + raise Exception(result.stderr) + logger.info("๐งช (6/%s) Unit testing...", STEPS) + partial_run( + ["pytest", "tests", "-m", "not integration"], + cwd=DIR_PLATFORM, + ) + logger.info("๐ญ (7/%s) Smoke testing...", STEPS) + # TODO: Improve smoke test coverage here + result = run( # noqa: S603 + [ # noqa: S603 + sys.executable, + "-c", + "from openbb import obb; obb.equity.price.historical('AAPL', provider='yfinance')", + ], + cwd=DIR_PLATFORM, + check=True, + capture_output=True, + text=True, + ) + if verbose: + logger.info("Captured result -> %s", result) + if result.stderr: + logger.error("โ stderr is not empty!") + raise Exception(result.stderr) + logger.info("๐ Great success! ๐") + logger.info( + "Confirm any files changed and run `poetry publish --build` from /openbb_platform" + ) if __name__ == "__main__": - msg = """ - You are about to publish a new version of OpenBB Platform to PyPI. - Please ensure you've read the "PUBLISH.md" file. - Also, please double check with `poetry config --list` if you're publishing to PyPI or TestPyPI. - """ + msg = ( + "\nYou are about to publish a new version of OpenBB Platform to PyPI." + + "\n\nPlease ensure you've read the PUBLISH.md file and double check with " + + "`poetry config --list` if you're publishing to PyPI or TestPyPI." + ) args = parse_args() - - res = input(f"{msg}\n\nDo you want to continue? [y/N] ") - + if not args.dry_run: + msg += "\n\n๐ You are NOT using the --dry-run flag!" + res = "y" if args.yes else input(f"{msg}\n\nDo you want to continue? [y/N] ") + print("") # noqa: T201 if res.lower() == "y": - publish(dry_run=args.dry_run, core=args.core, extensions=args.extensions) - - openbb_package_msg = """ - In order to publish the `openbb` package you need to manually update the - versions in the `pyproject.toml` file. Follow the steps below: - 1. Bump version: `poetry version prerelease --next-phase` - 2. Re-build the static assets that are bundled with the package - 3. Publish: `poetry publish --build` - """ - - print(openbb_package_msg) # noqa: T201 + logger.info("Process started. Press Ctrl+C to abort.") + try: + publish( + dry_run=args.dry_run, + core=args.core, + extensions=args.extensions, + openbb=args.openbb, + verbose=args.verbose, + semver=args.semver, + ) + except KeyboardInterrupt: + sys.exit(1) diff --git a/openbb_platform/core/openbb_core/app/static/utils/linters.py b/openbb_platform/core/openbb_core/app/static/utils/linters.py index 7f9162338bc..d22cf306169 100644 --- a/openbb_platform/core/openbb_core/app/static/utils/linters.py +++ b/openbb_platform/core/openbb_core/app/static/utils/linters.py @@ -36,10 +36,12 @@ class Linters: self.console.log(f"\n* {linter}") self.print_separator("^") - command = [linter] + list(self.directory.glob("*.py")) + command = [linter] if flags: command.extend(flags) # type: ignore - subprocess.run(command, check=False) # noqa: S603 + subprocess.run( + command + list(self.directory.glob("*.py")), check=False # noqa: S603 + ) self.print_separator("-") else: @@ -54,7 +56,7 @@ class Linters: def ruff(self): """Run ruff.""" - flags = ["--fix"] + flags = ["check", "--fix"] if not self.verbose and not Env().DEBUG_MODE: flags.append("--silent") self.run(linter="ruff", flags=flags) |