diff options
author | teh_coderer <me@tehcoderer.com> | 2022-11-30 19:35:51 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-11-30 20:35:51 -0500 |
commit | 2494342d4c8dee450441bcc8a8d6cac85d73ed73 (patch) | |
tree | 6e2cc0de052880954dfb39dd7f46bf7277ded2c8 | |
parent | 32bfba4874d0440ea3d978614d51ed5563ab40a7 (diff) |
[Docs] Fred Autogen fix (#3658)
* Update controller_doc_classes.py
* test gh-actions
* Update gh-pages.yml
* Update generate_terminal_markdown.py
* cleanup generate_terminal_markdown.py, patch binance error preventing generation
Co-authored-by: andrewkenreich <andrew.kenreich@gmail.com>
Co-authored-by: James Maslek <jmaslek11@gmail.com>
-rw-r--r-- | .github/workflows/gh-pages.yml | 4 | ||||
-rw-r--r-- | openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py | 45 | ||||
-rw-r--r-- | website/controller_doc_classes.py | 3 | ||||
-rw-r--r-- | website/generate_terminal_markdown.py | 210 |
4 files changed, 185 insertions, 77 deletions
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 6d913a6a20a..ff320231b5f 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -13,10 +13,10 @@ jobs: - name: Git checkout uses: actions/checkout@v3 # actions/checkout v3.0.2 - - name: Setup Python 3.10 + - name: Setup Python 3.9 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" architecture: x64 - name: Install Poetry diff --git a/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py b/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py index abb14ad0347..64dd7550fa2 100644 --- a/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py +++ b/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py @@ -2,8 +2,10 @@ __docformat__ = "numpy" from typing import Any, Dict, List + import ccxt import pandas as pd + from openbb_terminal.cryptocurrency.dataframe_helpers import prettify_column_names @@ -37,10 +39,45 @@ def get_binance_currencies() -> List[str]: # Refactor this eventually to allow for any entered exchange - # right now only works on default binace for "ob" and "trades" - exchange = ccxt.binance({"fetchCurrencies": True}) - exchange.load_markets() - currencies = exchange.quoteCurrencies - return [c["code"] for c in currencies.values()] + + # Commented out for now, since binance started blocking requests + # exchange = ccxt.binance({"fetchCurrencies": True}) + # exchange.load_markets() + # currencies = exchange.quoteCurrencies + # return [c["code"] for c in currencies.values()] + return [ + "AUD", + "BIDR", + "BKRW", + "BNB", + "BRL", + "BTC", + "BUSD", + "BVND", + "DAI", + "DOGE", + "DOT", + "ETH", + "EUR", + "GBP", + "IDRT", + "NGN", + "PAX", + "PLN", + "RUB", + "TRX", + "TRY", + "TUSD", + "UAH", + "USDC", + "USDP", + "USDS", + "USDT", + "UST", + "VAI", + "XRP", + "ZAR", + ] def get_orderbook(exchange: str, symbol: str, to_symbol: str) -> Dict[str, Any]: diff --git a/website/controller_doc_classes.py b/website/controller_doc_classes.py index fe723278d19..404ed2b927f 100644 --- a/website/controller_doc_classes.py +++ b/website/controller_doc_classes.py @@ -13,8 +13,11 @@ import pandas as pd from pandas.tseries.holiday import USFederalHolidayCalendar import openbb_terminal +from openbb_terminal.decorators import disable_check_api from openbb_terminal.parent_classes import BaseController, CryptoBaseController +disable_check_api() + DF_STOCK = pd.DataFrame.from_dict( data={ pd.Timestamp("2020-11-30 00:00:00"): { diff --git a/website/generate_terminal_markdown.py b/website/generate_terminal_markdown.py index 181805df5d5..358c3b9a1e3 100644 --- a/website/generate_terminal_markdown.py +++ b/website/generate_terminal_markdown.py @@ -1,11 +1,13 @@ import json import os +import re import shutil import traceback from datetime import datetime from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, List, Union +from openbb_terminal.core.config.paths import USER_DATA_DIRECTORY from openbb_terminal.rich_config import console from website.controller_doc_classes import ( ControllerDoc, @@ -14,28 +16,42 @@ from website.controller_doc_classes import ( ) website_path = Path(__file__).parent.absolute() +USER_PATH = (f"{USER_DATA_DIRECTORY}", "`USER_DATA_DIRECTORY`") def existing_markdown_file_examples( - ctrl: ControllerDoc, cat: Dict[str, str] -) -> Dict[str, Optional[Union[str, List[str]]]]: - """Get existing markdown file examples""" - trail = ctrl.trailmap.split(".") + trailmap: str, cmd_dict: Dict[str, str] +) -> Dict[str, Union[str, List[str]]]: + """Get existing markdown file examples + + Parameters + ---------- + trailmap : str + The trailmap to the function + cmd_dict : Dict[str, str] + The dictionary of the command + + Returns + ------- + Dict[str, Union[str, List[str]]] + Updated dictionary of the command with examples and images keys + """ + trail = trailmap.split(".") for sub in trail: if sub in ["ba", "ta", "qa"]: - trail.remove(ctrl.trailmap.split(".")[0]) + trail.remove(trailmap.split(".")[0]) examples_path = ( - f"old_content/terminal/{'/'.join(trail)}/{cat['cmd_name']}/_index.md" + f"old_content/terminal/{'/'.join(trail)}/{cmd_dict['cmd_name']}/_index.md" ) - examples_dict: Dict[str, Optional[Union[str, List[str]]]] = {} + examples_dict: Dict[str, Union[str, List[str]]] = {} if os.path.exists(website_path / examples_path): with open(website_path / examples_path, encoding="utf-8") as f: content = f.read() - examples: Optional[str] = None + examples: str = "" if "Example:" in content: example_split = content.split("Example:")[1].split("```") if example_split and len(example_split) > 1: @@ -51,8 +67,8 @@ def existing_markdown_file_examples( # pylint: disable=isinstance-second-argument-not-valid-type -def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]: - """Get commands and parsers from ControllerDoc""" +def process_cmd_parsers(ctrl: ControllerDoc) -> List[Dict[str, str]]: + """Process command parsers from ControllerDoc""" commands = [] for cmd, parser in ctrl.cmd_parsers.items(): @@ -62,54 +78,59 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]: if action.dest == "help": continue - default = action.default - if default is not None: + if (default := action.default) is not None: + if isinstance(default, list): default = ", ".join([str(x) for x in default]) + elif isinstance(default, datetime): if "start" in action.dest: default = "datetime.now() - timedelta(days=365)" elif "end" in action.dest or "date" in action.dest: default = "datetime.now()" - choices = action.choices - if choices is not None: + if (choices := action.choices) is not None: - if isinstance(choices, list): - listdict = [] + # We do this so that range(0, N) objects are not converted to a list + if isinstance(choices, (dict, type({}.keys()), list)): + + listdict: list = [] for choice in choices: + # We check for nested dicts if isinstance(choice, (dict, type({}.keys()))): listdict.append([f"{k}" for k in choice]) - if listdict: - choices = listdict - else: - choices = [f"{x}" for x in choices] - choices = ", ".join(choices) if len(choices) > 0 else None + choices = listdict or [f"{x}" for x in choices] + choices = ", ".join(choices) if len(choices) > 0 else "None" + + if action.dest == "file": + new_desc = "File in `EXPORTS` or `CUSTOM_IMPORTS` directories" - elif isinstance(choices, (dict, type({}.keys()))): - choices = [f"{k}" for k in choices] - choices = ", ".join(choices) if len(choices) > 0 else None + if (file_ext := re.compile(r"\.(\w{3,5})").findall(choices)) != []: + exts = set(file_ext) + ext_examples = "file_name." + ", file_name.".join(exts) + new_desc += f" (e.g: `{ext_examples}`)" + action.choices = [f"`{ext_examples}`"] - doc = action.help - if doc is not None: + choices = new_desc + + if (doc := action.help) is not None: # We do this to fix multiline docstrings for the markdown - doc = " ".join(doc.split()) + doc = " ".join(doc.split()).replace(*USER_PATH) actions.append( { "opt_name": action.dest if action.dest else "", "doc": doc if doc else "", - "default": default, + "default": f"{default}".replace(*USER_PATH), "optional": not action.required, "choices": choices, } ) - desc = parser.description - if desc is not None: + if (desc := parser.description) is not None: # We do this to fix multiline docstrings for the markdown - desc = " ".join(desc.split()) + desc = " ".join(desc.split()).replace(*USER_PATH) param = { "cmd_name": cmd.replace("call_", ""), @@ -119,37 +140,45 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]: } commands.append(param) - return {"category_name": ctrl.name, "cmds": commands} + return commands + +def generate_markdown( + cmd_meta: Dict[str, str], examples: Dict[str, Union[str, List[str]]] +) -> str: + """Generate markdown section -def generate_markdown(cmd_meta: Dict[str, str], examples: Dict[str, str]): - """Generate markdown string""" + Parameters + ---------- + cmd_meta : Dict[str, str] + Command metadata + examples : Dict[str, Union[str, List[str]]] + Command examples + + Returns + ------- + str + Markdown string + """ if not cmd_meta: raise ValueError("No command metadata found") markdown = f"""--- +########### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ########### title: {cmd_meta["cmd_name"]} description: OpenBB Terminal Function ---\n\n""" - markdown += generate_markdown_section(cmd_meta, examples) - - return markdown - - -def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) -> str: - """Generate markdown section""" - # head meta https://docusaurus.io/docs/markdown-features/head-metadata - markdown = f"# {meta['cmd_name']}\n\n{meta['description']}\n\n" - markdown += f"### Usage\n\n```python\n{meta['usage']}\n```\n\n" + markdown += f"# {cmd_meta['cmd_name']}\n\n{cmd_meta['description']}\n\n" + markdown += f"### Usage\n\n```python\n{cmd_meta['usage']}\n```\n\n" markdown += "---\n\n## Parameters\n\n" - if meta["actions"]: + if cmd_meta["actions"]: markdown += "| Name | Description | Default | Optional | Choices |\n" markdown += "| ---- | ----------- | ------- | -------- | ------- |\n" - for param in meta["actions"]: + for param in cmd_meta["actions"]: if isinstance(param, dict): markdown += ( f"| {param['opt_name']} | {param['doc']} | {param['default']} " @@ -158,13 +187,13 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) -> else: markdown += "This command has no parameters\n\n" - if examples.get("example", None): + if examples.get("example", ""): markdown += "\n\n---\n\n## Examples\n\n" markdown += f"```python\n{examples['example']}\n```" markdown += "\n" - if examples.get("images", []): + if isinstance(examples.get("images", None), list): for image in examples["images"]: markdown += f"{image}\n\n" @@ -173,7 +202,25 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) -> def add_todict(d: dict, location_path: list, cmd_name: str, full_path: str) -> dict: - """Adds the trailmap to the dictionary. This function creates the dictionary paths to the function.""" + """Adds the trailmap to the dictionary. This function creates the dictionary paths to the function. + + + Parameters + ---------- + d : dict + The dictionary to add the path to + location_path : list + The path to the function + cmd_name : str + The name of the function + full_path : str + The full path to the function + + Returns + ------- + dict + The updated dictionary + """ if location_path[0] not in d: d[location_path[0]] = {} @@ -196,13 +243,13 @@ def main() -> bool: load_ctrls = LoadControllersDoc() ctrls = load_ctrls.available_controllers() - kwargs = {"encoding": "utf-8", "newline": "\n"} + wopen_kwargs = {"encoding": "utf-8", "newline": "\n"} console.print( "[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]" ) - content_path = website_path / "content/terminal/reference" - terminal_ref = {} + content_path: Path = website_path / "content/terminal/reference" + terminal_ref: Dict[str, dict] = {} for file in content_path.glob("*"): if file.is_file(): @@ -210,30 +257,32 @@ def main() -> bool: else: shutil.rmtree(file) - for ctrlstr in ctrls: + for ctrl_trailmap in ctrls: try: - ctrl = load_ctrls.get_controller_doc(ctrlstr) - cmd_meta = get_parser(ctrl) + ctrl = load_ctrls.get_controller_doc(ctrl_trailmap) + ctrl_cmds = process_cmd_parsers(ctrl) - for cat in cmd_meta["cmds"]: - examples = existing_markdown_file_examples(ctrl, cat) - markdown = generate_markdown(cat, examples) + for cmd in ctrl_cmds: + examples = existing_markdown_file_examples(ctrl_trailmap, cmd) + markdown = generate_markdown(cmd, examples) - if cat["cmd_name"] == "index": - cat["cmd_name"] = "index_cmd" + if cmd["cmd_name"] == "index": + cmd["cmd_name"] = "index_cmd" - trail = ctrl.trailmap.split(".") + trail = ctrl_trailmap.split(".") - terminal_ref = add_todict(terminal_ref, trail, cat["cmd_name"], trail) - filepath = f"{str(content_path)}/{'/'.join(trail)}/{cat['cmd_name']}.md" + terminal_ref = add_todict(terminal_ref, trail, cmd["cmd_name"], trail) + filepath = content_path / f"{'/'.join(trail)}/{cmd['cmd_name']}.md" - os.makedirs(os.path.dirname(filepath), exist_ok=True) - with open(filepath, "w", **kwargs) as f: + filepath.parent.mkdir(parents=True, exist_ok=True) + with open(filepath, "w", **wopen_kwargs) as f: # type: ignore f.write(markdown) except Exception as e: traceback.print_exc() - console.print(f"[red]Failed to generate markdown for {ctrlstr}: {e}[/red]") + console.print( + f"[red]Failed to generate markdown for {ctrl_trailmap}: {e}[/red]" + ) return False terminal_ref = { @@ -241,10 +290,12 @@ def main() -> bool: for k, v in sorted(terminal_ref.items(), key=lambda item: item[0]) } - with open(content_path / "_category_.json", "w", **kwargs) as f: + # Generate root "_category_.json" file + with open(content_path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore f.write(json.dumps({"label": "Terminal Reference", "position": 4}, indent=2)) - with open(content_path / "index.md", "w", **kwargs) as f: + # Generate root "index.md" file + with open(content_path / "index.md", "w", **wopen_kwargs) as f: # type: ignore f.write( f"# OpenBB Terminal Features\n\n{generate_index_markdown('', terminal_ref, 2)}" ) @@ -252,7 +303,7 @@ def main() -> bool: def gen_category_json(fname: str, path: Path): """Generate category json""" fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title() - with open(path / "_category_.json", "w", **kwargs) as f: + with open(path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore f.write(json.dumps({"label": fname}, indent=2)) def gen_category_recursive(nested_path: Path): @@ -262,6 +313,7 @@ def main() -> bool: gen_category_json(folder.name, folder) gen_category_recursive(folder) # pylint: disable=cell-var-from-loop + # Generate modules/sub category json and index files gen_category_recursive(content_path) console.print( @@ -271,7 +323,23 @@ def main() -> bool: return True -def generate_index_markdown(markdown, d, level): +def generate_index_markdown(markdown: str, d: dict, level: int) -> str: + """Generate index markdown + + Parameters + ---------- + markdown : str + The markdown to add to + d : dict + The dictionary to recursively generate markdown from + level : int + The level of the markdown header + + Returns + ------- + str + Generated index file markdown string + """ for key in d: if isinstance(d[key], dict): markdown += f"\n{'#' * level} {key}\n\n" |