From bd75a9746f2643594558f67f710b743caaac9802 Mon Sep 17 00:00:00 2001 From: montezdesousa <79287829+montezdesousa@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:18:00 +0100 Subject: Move files used in gh actions from root (#6346) * Move files used in gh actions from root * keep this * pydocstyle * ^ * fix: relative path, os independent * use relative root path * ^ * move noxfile.py to .github/scripts * remove flag --- .github/scripts/noxfile.py | 29 ++++ .github/scripts/process_changelog.py | 82 ++++++++++ .github/scripts/summarize_changelog.py | 166 +++++++++++++++++++++ .github/workflows/draft-release.yml | 4 +- .github/workflows/platform-core.yml | 2 +- noxfile.py | 25 ---- .../tests/utils/integration_tests_generator.py | 6 +- openbb_platform/tests/test_pyproject_toml.py | 7 +- process_changelog.py | 82 ---------- summarize_changelog.py | 166 --------------------- 10 files changed, 289 insertions(+), 280 deletions(-) create mode 100644 .github/scripts/noxfile.py create mode 100644 .github/scripts/process_changelog.py create mode 100644 .github/scripts/summarize_changelog.py delete mode 100644 noxfile.py delete mode 100644 process_changelog.py delete mode 100644 summarize_changelog.py diff --git a/.github/scripts/noxfile.py b/.github/scripts/noxfile.py new file mode 100644 index 00000000000..ed4411601c5 --- /dev/null +++ b/.github/scripts/noxfile.py @@ -0,0 +1,29 @@ +"""Nox sessions.""" + +from pathlib import Path + +import nox + +ROOT_DIR = Path(__file__).parent.parent.parent +PLATFORM_DIR = ROOT_DIR / "openbb_platform" +PLATFORM_TESTS = [ + str(PLATFORM_DIR / p) for p in ["tests", "core", "providers", "extensions"] +] + + +@nox.session(python=["3.9", "3.10", "3.11"]) +def tests(session): + """Run the test suite.""" + session.install("poetry", "toml") + session.run( + "python", + str(PLATFORM_DIR / "dev_install.py"), + "-e", + "all", + external=True, + ) + session.install("pytest") + session.install("pytest-cov") + session.run( + "pytest", *PLATFORM_TESTS, f"--cov={PLATFORM_DIR}", "-m", "not integration" + ) diff --git a/.github/scripts/process_changelog.py b/.github/scripts/process_changelog.py new file mode 100644 index 00000000000..fbdfdf9ce0b --- /dev/null +++ b/.github/scripts/process_changelog.py @@ -0,0 +1,82 @@ +# process_changelog.py +import logging +import re +import sys + +# Set up basic configuration for logging +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + +def process_changelog(file_path, release_pr_number): + # Attempt to open and read the file content + try: + with open(file_path) as file: # Default mode is 'r' for read + lines = file.readlines() + except OSError as e: # Catching file I/O errors + logging.error(f"Failed to open or read file: {e}") + return + + pr_occurrences = {} # Dictionary to track occurrences of PR numbers + + # Iterate through each line to find PR numbers + for i, line in enumerate(lines): + match = re.search(r"\(#(\d+)\)", line) # Regex to find PR numbers + if match: + pr_number = int(match.group(1)) + # Add line index to the list of occurrences for the PR number + if pr_number not in pr_occurrences: + pr_occurrences[pr_number] = [] + pr_occurrences[pr_number].append(i) + + # Set of indices to remove: includes all but last occurrence of each PR number + to_remove = { + i + for pr, indices in pr_occurrences.items() + if len(indices) > 1 + for i in indices[:-1] + } + # Also remove any PR entries less than or equal to the specified release PR number + to_remove.update( + i + for pr, indices in pr_occurrences.items() + for i in indices + if pr <= release_pr_number + ) + + # Filter out lines marked for removal + processed_lines = [line for i, line in enumerate(lines) if i not in to_remove] + + # Final sweep: Ensure no missed duplicates, keeping only the last occurrence + final_lines = [] + seen_pr_numbers = set() # Track seen PR numbers to identify duplicates + for line in reversed( + processed_lines + ): # Start from the end to keep the last occurrence + match = re.search(r"\(#(\d+)\)", line) + if match: + pr_number = int(match.group(1)) + if pr_number in seen_pr_numbers: + continue # Skip duplicate entries + seen_pr_numbers.add(pr_number) + final_lines.append(line) + final_lines.reverse() # Restore original order + + # Write the processed lines back to the file + try: + with open(file_path, "w") as file: + file.writelines(final_lines) + except OSError as e: # Handling potential write errors + logging.error(f"Failed to write to file: {e}") + + +if __name__ == "__main__": + # Ensure correct command line arguments + if len(sys.argv) < 3: + logging.error( + "Usage: python process_changelog.py " + ) + sys.exit(1) + + file_path = sys.argv[1] + release_pr_number = int(sys.argv[2]) + process_changelog(file_path, release_pr_number) diff --git a/.github/scripts/summarize_changelog.py b/.github/scripts/summarize_changelog.py new file mode 100644 index 00000000000..75648dff105 --- /dev/null +++ b/.github/scripts/summarize_changelog.py @@ -0,0 +1,166 @@ +"""Changelog v2 summary generator.""" + +import logging +import re +import sys +from typing import Dict + +import requests + + +def fetch_pr_details(owner: str, repo: str, pr_number: str, github_token: str) -> dict: + """Fetch details of a specific PR from GitHub.""" + url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + headers = {"Authorization": f"token {github_token}"} + response = requests.get(url, headers=headers, timeout=10) + if response.status_code == 200: + return response.json() + + logging.error( + "Failed to fetch PR details for PR #%s. Status code: %s", + pr_number, + response.status_code, + ) + return {} + + +def parse_and_fetch_pr_details( + markdown_text: str, owner: str, repo: str, github_token: str +) -> Dict[str, str]: + """Parse the markdown text and fetch details of PRs mentioned in the text.""" + sections = re.split(r"\n## ", markdown_text) + categories: Dict[str, str] = {} + + for section in sections: + split_section = section.split("\n", 1) + if len(split_section) < 2: + continue + + category_name = split_section[0].strip() + items_text = split_section[1].strip() + items = re.findall(r"- (?:\[.*?\] - )?(.*?) @.*? \(#(\d+)\)", items_text) + + for _, pr_number in items: + pr_details = fetch_pr_details(owner, repo, pr_number, github_token) + if pr_details: + try: + pr_info = { + "title": pr_details["title"], + "body": re.sub(r"\s+", " ", pr_details["body"].strip()).strip(), + } + except Exception as e: + logging.error( + "Failed to fetch PR details for PR #%s: %s", pr_number, e + ) + if category_name in categories: + categories[category_name].append(pr_info) # type: ignore + else: + categories[category_name] = [pr_info] # type: ignore + + return categories + + +def insert_summary_into_markdown( + markdown_text: str, category_name: str, summary: str +) -> str: + """Insert a summary into the markdown text directly under the specified category name.""" + marker = f"## {category_name}" + if marker in markdown_text: + # Find the position right after the category name + start_pos = markdown_text.find(marker) + len(marker) + # Find the position of the first newline after the category name to ensure we insert before any content + newline_pos = markdown_text.find("\n", start_pos) + if newline_pos != -1: + # Insert the summary right after the newline that follows the category name + # Ensuring it's on a new line and followed by two newlines before any subsequent content + updated_markdown = ( + markdown_text[: newline_pos + 1] + + "\n" + + summary + + markdown_text[newline_pos + 1 :] + ) + else: + # If there's no newline (e.g., end of file), just append the summary + updated_markdown = markdown_text + "\n\n" + summary + "\n" + return updated_markdown + + logging.error("Category '%s' not found in markdown.", category_name) + return markdown_text + + +def summarize_text_with_openai(text: str, openai_api_key: str) -> str: + """Summarize text using OpenAI's GPT model.""" + from openai import OpenAI # pylint: disable=C0415 + + openai = OpenAI(api_key=openai_api_key) + response = openai.chat.completions.create( + model="gpt-4", # noqa: E501 + messages=[ + { + "role": "system", + "content": "Summarize the following text in a concise way to describe what happened in the new release. This will be used on top of the changelog to provide a high-level overview of the changes. Make sure it is well-written, concise, structured and that it captures the essence of the text. It should read like a concise story.", # noqa: E501 # pylint: disable=C0301 + }, + {"role": "user", "content": text}, + ], + ) + return response.choices[0].message.content # type: ignore + + +def summarize_changelog_v2( + github_token: str, + openai_api_key: str, + owner: str = "OpenBB-finance", + repo: str = "OpenBBTerminal", + changelog_v2: str = "CHANGELOG.md", +) -> None: + """Summarize the Changelog v2 markdown text with PR details.""" + try: + with open(changelog_v2) as file: + logging.info("Reading file: %s", changelog_v2) + data = file.read() + except OSError as e: + logging.error("Failed to open or read file: %s", e) + return + + logging.info("Parsing and fetching PR details...") + categories = parse_and_fetch_pr_details(data, owner, repo, github_token) + + categories_of_interest = [ + "🚨 OpenBB Platform Breaking Changes", + "🦋 OpenBB Platform Enhancements", + "🐛 OpenBB Platform Bug Fixes", + "📚 OpenBB Documentation Changes", + ] + updated_markdown = data + + logging.info("Summarizing text with OpenAI...") + for category_of_interest in categories_of_interest: + if category_of_interest in categories: + pattern = r"\[.*?\]\(.*?\)|[*_`]" + aggregated_text = "\n".join( + [ + f"- {pr['title']}: {re.sub(pattern, '', pr['body'])}" # type: ignore + for pr in categories[category_of_interest] # type: ignore + ] + ) + summary = summarize_text_with_openai(aggregated_text, openai_api_key) + updated_markdown = insert_summary_into_markdown( + updated_markdown, category_of_interest, summary + ) + + with open(changelog_v2, "w") as file: + logging.info("Writing updated file: %s", changelog_v2) + file.write(updated_markdown) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + logging.error( + "Usage: python summarize_changelog.py " + ) + sys.exit(1) + + token = sys.argv[1] + openai_key = sys.argv[2] + + summarize_changelog_v2(github_token=token, openai_api_key=openai_key) diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml index 65c7d52dc66..3dab21e7812 100644 --- a/.github/workflows/draft-release.yml +++ b/.github/workflows/draft-release.yml @@ -36,8 +36,8 @@ jobs: - name: 🧬 Process Changelog run: | pip install requests openai - python process_changelog.py CHANGELOG.md ${{ github.event.inputs.release_pr_number }} - python summarize_changelog.py ${{ secrets.GITHUB_TOKEN }} ${{ secrets.OPENAI_API_KEY }} + python .github/scripts/process_changelog.py CHANGELOG.md ${{ github.event.inputs.release_pr_number }} + python .github/scripts/summarize_changelog.py ${{ secrets.GITHUB_TOKEN }} ${{ secrets.OPENAI_API_KEY }} cat CHANGELOG.md - name: 🛫 Create Release diff --git a/.github/workflows/platform-core.yml b/.github/workflows/platform-core.yml index c509c243c80..84ac783cfaf 100644 --- a/.github/workflows/platform-core.yml +++ b/.github/workflows/platform-core.yml @@ -43,4 +43,4 @@ jobs: - name: Run tests run: | pip install nox - nox -s tests --python ${{ matrix.python_version }} + nox -f .github/scripts/noxfile.py -s tests --python ${{ matrix.python_version }} diff --git a/noxfile.py b/noxfile.py deleted file mode 100644 index 02726297a91..00000000000 --- a/noxfile.py +++ /dev/null @@ -1,25 +0,0 @@ -import nox - -test_locations = [ - "openbb_platform/tests", - "openbb_platform/core", - "openbb_platform/providers", - "openbb_platform/extensions", -] - - -@nox.session(python=["3.9", "3.10", "3.11"]) -def tests(session): - session.install("poetry", "toml") - session.run( - "python", - "./openbb_platform/dev_install.py", - "-e", - "all", - external=True, - ) - session.install("pytest") - session.install("pytest-cov") - session.run( - "pytest", *test_locations, "--cov=openbb_platform/", "-m", "not integration" - ) diff --git a/openbb_platform/extensions/tests/utils/integration_tests_generator.py b/openbb_platform/extensions/tests/utils/integration_tests_generator.py index 7e1f478fd47..0e7663b03e4 100644 --- a/openbb_platform/extensions/tests/utils/integration_tests_generator.py +++ b/openbb_platform/extensions/tests/utils/integration_tests_generator.py @@ -20,6 +20,8 @@ from openbb_core.app.router import CommandMap from pydantic.fields import FieldInfo from pydantic_core import PydanticUndefined +ROOT_DIR = Path(__file__).parent.parent.parent.parent + TEST_TEMPLATE = """\n\n@parametrize( "params", [ @@ -43,9 +45,9 @@ def find_extensions(filter_chart: Optional[bool] = True): filter_ext = ["tests", "__pycache__"] if filter_chart: filter_ext.append("charting") - extensions = [x for x in Path("openbb_platform/extensions").iterdir() if x.is_dir()] + extensions = [x for x in (ROOT_DIR / "extensions").iterdir() if x.is_dir()] extensions.extend( - [x for x in Path("openbb_platform/obbject_extensions").iterdir() if x.is_dir()] + [x for x in (ROOT_DIR / "obbject_extensions").iterdir() if x.is_dir()] ) extensions = [x for x in extensions if x.name not in filter_ext] return extensions diff --git a/openbb_platform/tests/test_pyproject_toml.py b/openbb_platform/tests/test_pyproject_toml.py index 7012b0797dc..c4c6e908fc8 100644 --- a/openbb_platform/tests/test_pyproject_toml.py +++ b/openbb_platform/tests/test_pyproject_toml.py @@ -2,13 +2,16 @@ import glob import os +from pathlib import Path import toml +ROOT_DIR = Path(__file__).parent.parent + def test_optional_packages(): """Ensure only required extensions are built and versions respect pyproject.toml""" - data = toml.load("openbb_platform/pyproject.toml") + data = toml.load(ROOT_DIR / "pyproject.toml") dependencies = data["tool"]["poetry"]["dependencies"] extras = data["tool"]["poetry"]["extras"] all_packages = extras["all"] @@ -32,7 +35,7 @@ def test_optional_packages(): def test_default_package_files(): """Ensure only required extensions are built and versions respect pyproject.toml""" - data = toml.load("openbb_platform/pyproject.toml") + data = toml.load(Path(ROOT_DIR / "pyproject.toml")) dependencies = data["tool"]["poetry"]["dependencies"] package_files = glob.glob("openbb_platform/openbb/package/*.py") diff --git a/process_changelog.py b/process_changelog.py deleted file mode 100644 index fbdfdf9ce0b..00000000000 --- a/process_changelog.py +++ /dev/null @@ -1,82 +0,0 @@ -# process_changelog.py -import logging -import re -import sys - -# Set up basic configuration for logging -logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - -def process_changelog(file_path, release_pr_number): - # Attempt to open and read the file content - try: - with open(file_path) as file: # Default mode is 'r' for read - lines = file.readlines() - except OSError as e: # Catching file I/O errors - logging.error(f"Failed to open or read file: {e}") - return - - pr_occurrences = {} # Dictionary to track occurrences of PR numbers - - # Iterate through each line to find PR numbers - for i, line in enumerate(lines): - match = re.search(r"\(#(\d+)\)", line) # Regex to find PR numbers - if match: - pr_number = int(match.group(1)) - # Add line index to the list of occurrences for the PR number - if pr_number not in pr_occurrences: - pr_occurrences[pr_number] = [] - pr_occurrences[pr_number].append(i) - - # Set of indices to remove: includes all but last occurrence of each PR number - to_remove = { - i - for pr, indices in pr_occurrences.items() - if len(indices) > 1 - for i in indices[:-1] - } - # Also remove any PR entries less than or equal to the specified release PR number - to_remove.update( - i - for pr, indices in pr_occurrences.items() - for i in indices - if pr <= release_pr_number - ) - - # Filter out lines marked for removal - processed_lines = [line for i, line in enumerate(lines) if i not in to_remove] - - # Final sweep: Ensure no missed duplicates, keeping only the last occurrence - final_lines = [] - seen_pr_numbers = set() # Track seen PR numbers to identify duplicates - for line in reversed( - processed_lines - ): # Start from the end to keep the last occurrence - match = re.search(r"\(#(\d+)\)", line) - if match: - pr_number = int(match.group(1)) - if pr_number in seen_pr_numbers: - continue # Skip duplicate entries - seen_pr_numbers.add(pr_number) - final_lines.append(line) - final_lines.reverse() # Restore original order - - # Write the processed lines back to the file - try: - with open(file_path, "w") as file: - file.writelines(final_lines) - except OSError as e: # Handling potential write errors - logging.error(f"Failed to write to file: {e}") - - -if __name__ == "__main__": - # Ensure correct command line arguments - if len(sys.argv) < 3: - logging.error( - "Usage: python process_changelog.py " - ) - sys.exit(1) - - file_path = sys.argv[1] - release_pr_number = int(sys.argv[2]) - process_changelog(file_path, release_pr_number) diff --git a/summarize_changelog.py b/summarize_changelog.py deleted file mode 100644 index 75648dff105..00000000000 --- a/summarize_changelog.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Changelog v2 summary generator.""" - -import logging -import re -import sys -from typing import Dict - -import requests - - -def fetch_pr_details(owner: str, repo: str, pr_number: str, github_token: str) -> dict: - """Fetch details of a specific PR from GitHub.""" - url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" - headers = {"Authorization": f"token {github_token}"} - response = requests.get(url, headers=headers, timeout=10) - if response.status_code == 200: - return response.json() - - logging.error( - "Failed to fetch PR details for PR #%s. Status code: %s", - pr_number, - response.status_code, - ) - return {} - - -def parse_and_fetch_pr_details( - markdown_text: str, owner: str, repo: str, github_token: str -) -> Dict[str, str]: - """Parse the markdown text and fetch details of PRs mentioned in the text.""" - sections = re.split(r"\n## ", markdown_text) - categories: Dict[str, str] = {} - - for section in sections: - split_section = section.split("\n", 1) - if len(split_section) < 2: - continue - - category_name = split_section[0].strip() - items_text = split_section[1].strip() - items = re.findall(r"- (?:\[.*?\] - )?(.*?) @.*? \(#(\d+)\)", items_text) - - for _, pr_number in items: - pr_details = fetch_pr_details(owner, repo, pr_number, github_token) - if pr_details: - try: - pr_info = { - "title": pr_details["title"], - "body": re.sub(r"\s+", " ", pr_details["body"].strip()).strip(), - } - except Exception as e: - logging.error( - "Failed to fetch PR details for PR #%s: %s", pr_number, e - ) - if category_name in categories: - categories[category_name].append(pr_info) # type: ignore - else: - categories[category_name] = [pr_info] # type: ignore - - return categories - - -def insert_summary_into_markdown( - markdown_text: str, category_name: str, summary: str -) -> str: - """Insert a summary into the markdown text directly under the specified category name.""" - marker = f"## {category_name}" - if marker in markdown_text: - # Find the position right after the category name - start_pos = markdown_text.find(marker) + len(marker) - # Find the position of the first newline after the category name to ensure we insert before any content - newline_pos = markdown_text.find("\n", start_pos) - if newline_pos != -1: - # Insert the summary right after the newline that follows the category name - # Ensuring it's on a new line and followed by two newlines before any subsequent content - updated_markdown = ( - markdown_text[: newline_pos + 1] - + "\n" - + summary - + markdown_text[newline_pos + 1 :] - ) - else: - # If there's no newline (e.g., end of file), just append the summary - updated_markdown = markdown_text + "\n\n" + summary + "\n" - return updated_markdown - - logging.error("Category '%s' not found in markdown.", category_name) - return markdown_text - - -def summarize_text_with_openai(text: str, openai_api_key: str) -> str: - """Summarize text using OpenAI's GPT model.""" - from openai import OpenAI # pylint: disable=C0415 - - openai = OpenAI(api_key=openai_api_key) - response = openai.chat.completions.create( - model="gpt-4", # noqa: E501 - messages=[ - { - "role": "system", - "content": "Summarize the following text in a concise way to describe what happened in the new release. This will be used on top of the changelog to provide a high-level overview of the changes. Make sure it is well-written, concise, structured and that it captures the essence of the text. It should read like a concise story.", # noqa: E501 # pylint: disable=C0301 - }, - {"role": "user", "content": text}, - ], - ) - return response.choices[0].message.content # type: ignore - - -def summarize_changelog_v2( - github_token: str, - openai_api_key: str, - owner: str = "OpenBB-finance", - repo: str = "OpenBBTerminal", - changelog_v2: str = "CHANGELOG.md", -) -> None: - """Summarize the Changelog v2 markdown text with PR details.""" - try: - with open(changelog_v2) as file: - logging.info("Reading file: %s", changelog_v2) - data = file.read() - except OSError as e: - logging.error("Failed to open or read file: %s", e) - return - - logging.info("Parsing and fetching PR details...") - categories = parse_and_fetch_pr_details(data, owner, repo, github_token) - - categories_of_interest = [ - "🚨 OpenBB Platform Breaking Changes", - "🦋 OpenBB Platform Enhancements", - "🐛 OpenBB Platform Bug Fixes", - "📚 OpenBB Documentation Changes", - ] - updated_markdown = data - - logging.info("Summarizing text with OpenAI...") - for category_of_interest in categories_of_interest: - if category_of_interest in categories: - pattern = r"\[.*?\]\(.*?\)|[*_`]" - aggregated_text = "\n".join( - [ - f"- {pr['title']}: {re.sub(pattern, '', pr['body'])}" # type: ignore - for pr in categories[category_of_interest] # type: ignore - ] - ) - summary = summarize_text_with_openai(aggregated_text, openai_api_key) - updated_markdown = insert_summary_into_markdown( - updated_markdown, category_of_interest, summary - ) - - with open(changelog_v2, "w") as file: - logging.info("Writing updated file: %s", changelog_v2) - file.write(updated_markdown) - - -if __name__ == "__main__": - if len(sys.argv) < 3: - logging.error( - "Usage: python summarize_changelog.py " - ) - sys.exit(1) - - token = sys.argv[1] - openai_key = sys.argv[2] - - summarize_changelog_v2(github_token=token, openai_api_key=openai_key) -- cgit v1.2.3