[Docs] Fred Autogen fix (#3658)

* Update controller_doc_classes.py

* test gh-actions

* Update gh-pages.yml

* Update generate_terminal_markdown.py

* cleanup generate_terminal_markdown.py, patch binance error preventing generation

Co-authored-by: andrewkenreich <andrew.kenreich@gmail.com>
Co-authored-by: James Maslek <jmaslek11@gmail.com>

4 files changed, 185 insertions, 77 deletions

diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
index 6d913a6a20a..ff320231b5f 100644
--- a/.github/workflows/gh-pages.yml
+++ b/.github/workflows/gh-pages.yml
@@ -13,10 +13,10 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v3 # actions/checkout v3.0.2
 
-      - name: Setup Python 3.10
+      - name: Setup Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.9"
           architecture: x64
 
       - name: Install Poetry
diff --git a/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py b/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py
index abb14ad0347..64dd7550fa2 100644
--- a/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py
+++ b/openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py
@@ -2,8 +2,10 @@
 __docformat__ = "numpy"
 
 from typing import Any, Dict, List
+
 import ccxt
 import pandas as pd
+
 from openbb_terminal.cryptocurrency.dataframe_helpers import prettify_column_names
 
 
@@ -37,10 +39,45 @@ def get_binance_currencies() -> List[str]:
 
     # Refactor this eventually to allow for any entered exchange -
     # right now only works on default binace for "ob" and "trades"
-    exchange = ccxt.binance({"fetchCurrencies": True})
-    exchange.load_markets()
-    currencies = exchange.quoteCurrencies
-    return [c["code"] for c in currencies.values()]
+
+    # Commented out for now, since binance started blocking requests
+    # exchange = ccxt.binance({"fetchCurrencies": True})
+    # exchange.load_markets()
+    # currencies = exchange.quoteCurrencies
+    # return [c["code"] for c in currencies.values()]
+    return [
+        "AUD",
+        "BIDR",
+        "BKRW",
+        "BNB",
+        "BRL",
+        "BTC",
+        "BUSD",
+        "BVND",
+        "DAI",
+        "DOGE",
+        "DOT",
+        "ETH",
+        "EUR",
+        "GBP",
+        "IDRT",
+        "NGN",
+        "PAX",
+        "PLN",
+        "RUB",
+        "TRX",
+        "TRY",
+        "TUSD",
+        "UAH",
+        "USDC",
+        "USDP",
+        "USDS",
+        "USDT",
+        "UST",
+        "VAI",
+        "XRP",
+        "ZAR",
+    ]
 
 
 def get_orderbook(exchange: str, symbol: str, to_symbol: str) -> Dict[str, Any]:
diff --git a/website/controller_doc_classes.py b/website/controller_doc_classes.py
index fe723278d19..404ed2b927f 100644
--- a/website/controller_doc_classes.py
+++ b/website/controller_doc_classes.py
@@ -13,8 +13,11 @@ import pandas as pd
 from pandas.tseries.holiday import USFederalHolidayCalendar
 
 import openbb_terminal
+from openbb_terminal.decorators import disable_check_api
 from openbb_terminal.parent_classes import BaseController, CryptoBaseController
 
+disable_check_api()
+
 DF_STOCK = pd.DataFrame.from_dict(
     data={
         pd.Timestamp("2020-11-30 00:00:00"): {
diff --git a/website/generate_terminal_markdown.py b/website/generate_terminal_markdown.py
index 181805df5d5..358c3b9a1e3 100644
--- a/website/generate_terminal_markdown.py
+++ b/website/generate_terminal_markdown.py
@@ -1,11 +1,13 @@
 import json
 import os
+import re
 import shutil
 import traceback
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Union
 
+from openbb_terminal.core.config.paths import USER_DATA_DIRECTORY
 from openbb_terminal.rich_config import console
 from website.controller_doc_classes import (
     ControllerDoc,
@@ -14,28 +16,42 @@ from website.controller_doc_classes import (
 )
 
 website_path = Path(__file__).parent.absolute()
+USER_PATH = (f"{USER_DATA_DIRECTORY}", "`USER_DATA_DIRECTORY`")
 
 
 def existing_markdown_file_examples(
-    ctrl: ControllerDoc, cat: Dict[str, str]
-) -> Dict[str, Optional[Union[str, List[str]]]]:
-    """Get existing markdown file examples"""
-    trail = ctrl.trailmap.split(".")
+    trailmap: str, cmd_dict: Dict[str, str]
+) -> Dict[str, Union[str, List[str]]]:
+    """Get existing markdown file examples
+
+    Parameters
+    ----------
+    trailmap : str
+        The trailmap to the function
+    cmd_dict : Dict[str, str]
+        The dictionary of the command
+
+    Returns
+    -------
+    Dict[str, Union[str, List[str]]]
+        Updated dictionary of the command with examples and images keys
+    """
+    trail = trailmap.split(".")
     for sub in trail:
         if sub in ["ba", "ta", "qa"]:
-            trail.remove(ctrl.trailmap.split(".")[0])
+            trail.remove(trailmap.split(".")[0])
 
     examples_path = (
-        f"old_content/terminal/{'/'.join(trail)}/{cat['cmd_name']}/_index.md"
+        f"old_content/terminal/{'/'.join(trail)}/{cmd_dict['cmd_name']}/_index.md"
     )
-    examples_dict: Dict[str, Optional[Union[str, List[str]]]] = {}
+    examples_dict: Dict[str, Union[str, List[str]]] = {}
 
     if os.path.exists(website_path / examples_path):
 
         with open(website_path / examples_path, encoding="utf-8") as f:
             content = f.read()
 
-            examples: Optional[str] = None
+            examples: str = ""
             if "Example:" in content:
                 example_split = content.split("Example:")[1].split("```")
                 if example_split and len(example_split) > 1:
@@ -51,8 +67,8 @@ def existing_markdown_file_examples(
 
 
 # pylint: disable=isinstance-second-argument-not-valid-type
-def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
-    """Get commands and parsers from ControllerDoc"""
+def process_cmd_parsers(ctrl: ControllerDoc) -> List[Dict[str, str]]:
+    """Process command parsers from ControllerDoc"""
 
     commands = []
     for cmd, parser in ctrl.cmd_parsers.items():
@@ -62,54 +78,59 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
             if action.dest == "help":
                 continue
 
-            default = action.default
-            if default is not None:
+            if (default := action.default) is not None:
+
                 if isinstance(default, list):
                     default = ", ".join([str(x) for x in default])
+
                 elif isinstance(default, datetime):
                     if "start" in action.dest:
                         default = "datetime.now() - timedelta(days=365)"
                     elif "end" in action.dest or "date" in action.dest:
                         default = "datetime.now()"
 
-            choices = action.choices
-            if choices is not None:
+            if (choices := action.choices) is not None:
 
-                if isinstance(choices, list):
-                    listdict = []
+                # We do this so that range(0, N) objects are not converted to a list
+                if isinstance(choices, (dict, type({}.keys()), list)):
+
+                    listdict: list = []
                     for choice in choices:
+                        # We check for nested dicts
                         if isinstance(choice, (dict, type({}.keys()))):
                             listdict.append([f"{k}" for k in choice])
 
-                    if listdict:
-                        choices = listdict
-                    else:
-                        choices = [f"{x}" for x in choices]
-                    choices = ", ".join(choices) if len(choices) > 0 else None
+                    choices = listdict or [f"{x}" for x in choices]
+                    choices = ", ".join(choices) if len(choices) > 0 else "None"
+
+                if action.dest == "file":
+                    new_desc = "File in `EXPORTS` or `CUSTOM_IMPORTS` directories"
 
-                elif isinstance(choices, (dict, type({}.keys()))):
-                    choices = [f"{k}" for k in choices]
-                    choices = ", ".join(choices) if len(choices) > 0 else None
+                    if (file_ext := re.compile(r"\.(\w{3,5})").findall(choices)) != []:
+                        exts = set(file_ext)
+                        ext_examples = "file_name." + ", file_name.".join(exts)
+                        new_desc += f" (e.g: `{ext_examples}`)"
+                        action.choices = [f"`{ext_examples}`"]
 
-            doc = action.help
-            if doc is not None:
+                    choices = new_desc
+
+            if (doc := action.help) is not None:
                 # We do this to fix multiline docstrings for the markdown
-                doc = " ".join(doc.split())
+                doc = " ".join(doc.split()).replace(*USER_PATH)
 
             actions.append(
                 {
                     "opt_name": action.dest if action.dest else "",
                     "doc": doc if doc else "",
-                    "default": default,
+                    "default": f"{default}".replace(*USER_PATH),
                     "optional": not action.required,
                     "choices": choices,
                 }
             )
 
-        desc = parser.description
-        if desc is not None:
+        if (desc := parser.description) is not None:
             # We do this to fix multiline docstrings for the markdown
-            desc = " ".join(desc.split())
+            desc = " ".join(desc.split()).replace(*USER_PATH)
 
         param = {
             "cmd_name": cmd.replace("call_", ""),
@@ -119,37 +140,45 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
         }
         commands.append(param)
 
-    return {"category_name": ctrl.name, "cmds": commands}
+    return commands
+
 
+def generate_markdown(
+    cmd_meta: Dict[str, str], examples: Dict[str, Union[str, List[str]]]
+) -> str:
+    """Generate markdown section
 
-def generate_markdown(cmd_meta: Dict[str, str], examples: Dict[str, str]):
-    """Generate markdown string"""
+    Parameters
+    ----------
+    cmd_meta : Dict[str, str]
+        Command metadata
+    examples : Dict[str, Union[str, List[str]]]
+        Command examples
+
+    Returns
+    -------
+    str
+        Markdown string
+    """
     if not cmd_meta:
         raise ValueError("No command metadata found")
 
     markdown = f"""---
+########### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ###########
 title: {cmd_meta["cmd_name"]}
 description: OpenBB Terminal Function
 ---\n\n"""
 
-    markdown += generate_markdown_section(cmd_meta, examples)
-
-    return markdown
-
-
-def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) -> str:
-    """Generate markdown section"""
-
     # head meta https://docusaurus.io/docs/markdown-features/head-metadata
-    markdown = f"# {meta['cmd_name']}\n\n{meta['description']}\n\n"
-    markdown += f"### Usage\n\n```python\n{meta['usage']}\n```\n\n"
+    markdown += f"# {cmd_meta['cmd_name']}\n\n{cmd_meta['description']}\n\n"
+    markdown += f"### Usage\n\n```python\n{cmd_meta['usage']}\n```\n\n"
 
     markdown += "---\n\n## Parameters\n\n"
-    if meta["actions"]:
+    if cmd_meta["actions"]:
         markdown += "| Name | Description | Default | Optional | Choices |\n"
         markdown += "| ---- | ----------- | ------- | -------- | ------- |\n"
 
-        for param in meta["actions"]:
+        for param in cmd_meta["actions"]:
             if isinstance(param, dict):
                 markdown += (
                     f"| {param['opt_name']} | {param['doc']} | {param['default']} "
@@ -158,13 +187,13 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) ->
     else:
         markdown += "This command has no parameters\n\n"
 
-    if examples.get("example", None):
+    if examples.get("example", ""):
         markdown += "\n\n---\n\n## Examples\n\n"
         markdown += f"```python\n{examples['example']}\n```"
 
     markdown += "\n"
 
-    if examples.get("images", []):
+    if isinstance(examples.get("images", None), list):
         for image in examples["images"]:
             markdown += f"{image}\n\n"
 
@@ -173,7 +202,25 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) ->
 
 
 def add_todict(d: dict, location_path: list, cmd_name: str, full_path: str) -> dict:
-    """Adds the trailmap to the dictionary. This function creates the dictionary paths to the function."""
+    """Adds the trailmap to the dictionary. This function creates the dictionary paths to the function.
+
+
+    Parameters
+    ----------
+    d : dict
+        The dictionary to add the path to
+    location_path : list
+        The path to the function
+    cmd_name : str
+        The name of the function
+    full_path : str
+        The full path to the function
+
+    Returns
+    -------
+    dict
+        The updated dictionary
+    """
 
     if location_path[0] not in d:
         d[location_path[0]] = {}
@@ -196,13 +243,13 @@ def main() -> bool:
 
     load_ctrls = LoadControllersDoc()
     ctrls = load_ctrls.available_controllers()
-    kwargs = {"encoding": "utf-8", "newline": "\n"}
+    wopen_kwargs = {"encoding": "utf-8", "newline": "\n"}
 
     console.print(
         "[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]"
     )
-    content_path = website_path / "content/terminal/reference"
-    terminal_ref = {}
+    content_path: Path = website_path / "content/terminal/reference"
+    terminal_ref: Dict[str, dict] = {}
 
     for file in content_path.glob("*"):
         if file.is_file():
@@ -210,30 +257,32 @@ def main() -> bool:
         else:
             shutil.rmtree(file)
 
-    for ctrlstr in ctrls:
+    for ctrl_trailmap in ctrls:
         try:
-            ctrl = load_ctrls.get_controller_doc(ctrlstr)
-            cmd_meta = get_parser(ctrl)
+            ctrl = load_ctrls.get_controller_doc(ctrl_trailmap)
+            ctrl_cmds = process_cmd_parsers(ctrl)
 
-            for cat in cmd_meta["cmds"]:
-                examples = existing_markdown_file_examples(ctrl, cat)
-                markdown = generate_markdown(cat, examples)
+            for cmd in ctrl_cmds:
+                examples = existing_markdown_file_examples(ctrl_trailmap, cmd)
+                markdown = generate_markdown(cmd, examples)
 
-                if cat["cmd_name"] == "index":
-                    cat["cmd_name"] = "index_cmd"
+                if cmd["cmd_name"] == "index":
+                    cmd["cmd_name"] = "index_cmd"
 
-                trail = ctrl.trailmap.split(".")
+                trail = ctrl_trailmap.split(".")
 
-                terminal_ref = add_todict(terminal_ref, trail, cat["cmd_name"], trail)
-                filepath = f"{str(content_path)}/{'/'.join(trail)}/{cat['cmd_name']}.md"
+                terminal_ref = add_todict(terminal_ref, trail, cmd["cmd_name"], trail)
+                filepath = content_path / f"{'/'.join(trail)}/{cmd['cmd_name']}.md"
 
-                os.makedirs(os.path.dirname(filepath), exist_ok=True)
-                with open(filepath, "w", **kwargs) as f:
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                with open(filepath, "w", **wopen_kwargs) as f:  # type: ignore
                     f.write(markdown)
 
         except Exception as e:
             traceback.print_exc()
-            console.print(f"[red]Failed to generate markdown for {ctrlstr}: {e}[/red]")
+            console.print(
+                f"[red]Failed to generate markdown for {ctrl_trailmap}: {e}[/red]"
+            )
             return False
 
     terminal_ref = {
@@ -241,10 +290,12 @@ def main() -> bool:
         for k, v in sorted(terminal_ref.items(), key=lambda item: item[0])
     }
 
-    with open(content_path / "_category_.json", "w", **kwargs) as f:
+    # Generate root "_category_.json" file
+    with open(content_path / "_category_.json", "w", **wopen_kwargs) as f:  # type: ignore
         f.write(json.dumps({"label": "Terminal Reference", "position": 4}, indent=2))
 
-    with open(content_path / "index.md", "w", **kwargs) as f:
+    # Generate root "index.md" file
+    with open(content_path / "index.md", "w", **wopen_kwargs) as f:  # type: ignore
         f.write(
             f"# OpenBB Terminal Features\n\n{generate_index_markdown('', terminal_ref, 2)}"
         )
@@ -252,7 +303,7 @@ def main() -> bool:
     def gen_category_json(fname: str, path: Path):
         """Generate category json"""
         fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title()
-        with open(path / "_category_.json", "w", **kwargs) as f:
+        with open(path / "_category_.json", "w", **wopen_kwargs) as f:  # type: ignore
             f.write(json.dumps({"label": fname}, indent=2))
 
     def gen_category_recursive(nested_path: Path):
@@ -262,6 +313,7 @@ def main() -> bool:
                 gen_category_json(folder.name, folder)
                 gen_category_recursive(folder)  # pylint: disable=cell-var-from-loop
 
+    # Generate modules/sub category json and index files
     gen_category_recursive(content_path)
 
     console.print(
@@ -271,7 +323,23 @@ def main() -> bool:
     return True
 
 
-def generate_index_markdown(markdown, d, level):
+def generate_index_markdown(markdown: str, d: dict, level: int) -> str:
+    """Generate index markdown
+
+    Parameters
+    ----------
+    markdown : str
+        The markdown to add to
+    d : dict
+        The dictionary to recursively generate markdown from
+    level : int
+        The level of the markdown header
+
+    Returns
+    -------
+    str
+        Generated index file markdown string
+    """
     for key in d:
         if isinstance(d[key], dict):
             markdown += f"\n{'#' * level} {key}\n\n"