New Integration Tests (#3377)

47 files changed, 471 insertions, 397 deletions

diff --git a/.gitignore b/.gitignore
index fa4ee80a6fa..c5c1f458506 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,7 +31,8 @@ openbb_terminal/portfolio/portfolios/*
 .coverage.*
 !openbb_terminal/reports/templates/*.ipynb
 *_tests.csv
-integration_test_output/*
+integration_test_output/
+integration_test_summary/
 openbb_terminal/reports/templates/*.html
 website/functions
 *_sdk_audit.csv
diff --git a/openbb_terminal/integration_testing.py b/openbb_terminal/integration_testing.py
new file mode 100644
index 00000000000..ded2e8cb579
--- /dev/null
+++ b/openbb_terminal/integration_testing.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+"""Main Testing Module"""
+__docformat__ = "numpy"
+
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict
+import traceback
+import argparse
+import logging
+import csv
+import sys
+import os
+
+from openbb_terminal.rich_config import console
+from openbb_terminal.core.config.paths import (
+    MISCELLANEOUS_DIRECTORY,
+    REPOSITORY_DIRECTORY,
+)
+from openbb_terminal.terminal_controller import run_scripts
+
+logger = logging.getLogger(__name__)
+special_arguments_values = [
+    "ticker",
+    "currency",
+    "crypto",
+    "country",
+    "repo",
+    "crypto_vs",
+    "crypto_full",
+    "currency_vs",
+]
+
+
+def build_test_path_list(path_list: List[str]) -> List[Path]:
+    """Build the paths to use in test mode."""
+    if path_list == "":
+        console.print("Please send a path when using test mode")
+        return []
+
+    test_files = []
+
+    for path in path_list:
+        script_path = MISCELLANEOUS_DIRECTORY / "scripts" / path
+
+        if script_path.exists():
+            chosen_path = script_path
+        else:
+            console.print(f"\n[red]Can't find the file: {script_path}[/red]\n")
+            continue
+
+        if chosen_path.is_file() and str(chosen_path).endswith(".openbb"):
+            test_files.append(str(chosen_path))
+        elif chosen_path.is_dir():
+            all_files = os.walk(script_path)
+            for root, _, files in all_files:
+                for name in files:
+                    if name.endswith(".openbb"):
+                        path_obj = f"{root}/{name}"
+                        test_files.append(path_obj)
+
+    test_files_unique = set(test_files)
+    return [Path(x) for x in test_files_unique]
+
+
+def format_failures(fails: dict, output: bool) -> None:
+    """Generates the message and csv from the fails dictionary
+
+    Parameters
+    -----------
+    fails: dict
+        The dictionary with failure information
+    output: bool
+        Whether or not to save output into a CSV file
+    """
+    if fails:
+        console.print("\n[red]Failures:[/red]\n")
+        for file, exception in fails.items():
+            logger.error("%s: %s failed", file, exception["exception"])
+        # Write results to CSV
+        if output:
+            timestamp = datetime.now().timestamp()
+            stamp_str = str(timestamp).replace(".", "")
+            whole_path = Path(REPOSITORY_DIRECTORY / "integration_test_summary")
+            whole_path.mkdir(parents=True, exist_ok=True)
+            output_path = f"{stamp_str}_tests.csv"
+            with open(whole_path / output_path, "w") as file:  # type: ignore
+                header = ["Script File", "Type", "Error", "Stacktrace"]
+                writer = csv.DictWriter(file, fieldnames=header)  # type: ignore
+                writer.writeheader()
+                for file, exception in fails.items():
+                    clean_type = (
+                        str(type(exception["exception"]))
+                        .replace("<class '", "")
+                        .replace("'>", "")
+                    )
+                    writer.writerow(
+                        {
+                            "Script File": file,
+                            "Type": clean_type,
+                            "Error": exception["exception"],
+                            "Stacktrace": exception["traceback"],
+                        }
+                    )
+
+            console.print(f"CSV of errors saved to {whole_path / output_path}")
+
+
+def run_test_list(
+    path_list: List[str], verbose: bool, special_arguments: Dict[str, str], output: bool
+):
+    """Run commands in test mode."""
+    os.environ["DEBUG_MODE"] = "true"
+
+    if not path_list:
+        path_list = [""]
+    test_files = build_test_path_list(path_list)
+    SUCCESSES = 0
+    FAILURES = 0
+    fails = {}
+    length = len(test_files)
+    i = 0
+    console.print("[green]OpenBB Terminal Integrated Tests:\n[/green]")
+    for file in test_files:
+        console.print(f"{((i/length)*100):.1f}%  {file}")
+        try:
+            run_scripts(
+                file,
+                test_mode=True,
+                verbose=verbose,
+                special_arguments=special_arguments,
+                output=output,
+            )
+            SUCCESSES += 1
+        except Exception as e:
+            _, _, exc_traceback = sys.exc_info()
+            fails[file] = {
+                "exception": e,
+                "traceback": traceback.extract_tb(exc_traceback),
+            }
+            FAILURES += 1
+        i += 1
+    format_failures(fails, output)
+    console.print(
+        f"Summary: [green]Successes: {SUCCESSES}[/green] [red]Failures: {FAILURES}[/red]"
+    )
+
+
+def parse_args_and_run():
+    """Parse input arguments and run integration tests."""
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        prog="testing",
+        description="Integration tests for the OpenBB Terminal.",
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        help=(
+            "The path or .openbb file to run. Starts at "
+            "OpenBBTerminal/openbb_terminal/miscellaneous/scripts"
+        ),
+        dest="path",
+        nargs="+",
+        default="",
+        type=str,
+    )
+    parser.add_argument(
+        "-t",
+        "--test",
+        action="store_true",
+        help=(
+            "Run the terminal in testing mode. Also run this option and '-h'"
+            " to see testing argument options."
+        ),
+    )
+    parser.add_argument(
+        "--no-output",
+        action="store_true",
+        default=False,
+        dest="no_output",
+        help="Blocks creation of CSV files and logs",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Enable verbose output for debugging",
+        dest="verbose",
+        action="store_true",
+        default=False,
+    )
+    # This is the list of special arguments a user can send
+    for arg in special_arguments_values:
+        parser.add_argument(
+            f"--{arg}",
+            help=f"Change the default values for {arg}",
+            dest=arg,
+            type=str,
+            default="",
+        )
+
+    ns_parser, _ = parser.parse_known_args()
+    special_args_dict = {x: getattr(ns_parser, x) for x in special_arguments_values}
+    run_test_list(
+        path_list=ns_parser.path,
+        verbose=ns_parser.verbose,
+        special_arguments=special_args_dict,
+        output=not ns_parser.no_output,
+    )
+
+
+if __name__ == "__main__":
+    parse_args_and_run()
diff --git a/openbb_terminal/miscellaneous/scripts/README.md b/openbb_terminal/miscellaneous/scripts/README.md
index 8598bfaa2a0..7fcdd1ad371 100644
--- a/openbb_terminal/miscellaneous/scripts/README.md
+++ b/openbb_terminal/miscellaneous/scripts/README.md
@@ -8,7 +8,7 @@ It aims to provide necessary information in order to:
 - Run `integration tests`
 - Identify errors noted from `integration tests`
 
-## 1.1. Why having integration tests ?
+## 1.1. Why have integration tests ?
 
 The purpose of integration tests is to provide standard usage examples that can be programmatically run
 to make sure that a specific functionality of the terminal and the process to utilize that functionality
@@ -28,6 +28,10 @@ should be `test_<menu>_<command>.openbb` if you are testing a specific command o
 if you are testing the entire menu. However, it is encouraged to create as specific of integration tests
 as possible to identify errors more precisely. Additionally, all tests must end with the `exit` command.
 
+These files can be given dynamic output with the following syntax `${key=default}`. Please note that
+both key and default can only contain letters and numbers with NO special characters. Each dynamic
+argument MUST contain a key and a default value.
+
 ### Examples
 
 Testing a specific command and it's arguments:
@@ -89,18 +93,19 @@ exit
 ### Conda Terminal
 
 After navigating to the location of the OpenBBTerminal repo, one can run integration tests in a
-few different ways using the wildcard expression.
+few different ways using the wildcard expression. Please include a `-t` with `terminal.py` to run
+the tests.
 
 - Run all integration tests:
 
     ```zsh
-    python terminal.py scripts/*.openbb -t
+    python terminal.py -t
     ```
 
 - Run some integration tests:
 
     ```zsh
-    python terminal.py scripts/test_stocks_*.openbb -t
+    python terminal.py -f stocks cryptocurrency -t
     ```
 
     *This specific example runs all of the stocks integration tests. One can use this same format for different tests.*
@@ -108,9 +113,25 @@ few different ways using the wildcard expression.
 - Run one integration tests:
 
     ```zsh
-    python terminal.py scripts/test_alt_covid.openbb -t
+    python terminal.py -f alternative/test_alt_covid.openbb -t
+    ```
+
+    *Note that the base path is `OpenBBTerminal/openbb_terminal/miscellaneous/scripts`.*
+
+- Run integration tests with arguments by adding --key=value
+
+    ```zsh
+    python terminal.py --ticker=aapl -t
     ```
 
+- To see a lot of possible keys, run the following:
+
+    ```zsh
+    python terminal.py -h -t
+    ```
+
+If there are any test failures a csv will be generated with detailed information on the failures.
+
 ### Installer Terminal
 
 Integration tests can also be used on installers, which is a packaged version of the conda terminal.
@@ -160,3 +181,5 @@ If the `-t` argument is not given, then the reason why a specific failure occurs
 test is printed inline while the test is being run.
 
 If there is an error, one can identify the command and or series of steps that causes it fairly easily.
+
+Output from the integration tests can also be viewed in the `integration_test_output` folder .
diff --git a/openbb_terminal/miscellaneous/scripts/alternative/test_alt_covid.openbb b/openbb_terminal/miscellaneous/scripts/alternative/test_alt_covid.openbb
index 76d0f10aa45..dabed0b273d 100644
--- a/openbb_terminal/miscellaneous/scripts/alternative/test_alt_covid.openbb
+++ b/openbb_terminal/miscellaneous/scripts/alternative/test_alt_covid.openbb
@@ -1,14 +1,10 @@
 alternative
 covid
-country Australia
+country ${country=Australia}
 reset
 slopes
-country US
 ov
-country Russia
 deaths
-country United Kingdom
 cases
-country Canada
 rates
-exit
\ No newline at end of file
+exit
diff --git a/openbb_terminal/miscellaneous/scripts/alternative/test_alt_oss.openbb b/openbb_terminal/miscellaneous/scripts/alternative/test_alt_oss.openbb
index 1973e1d2dd1..3ac0100b291 100644
--- a/openbb_terminal/miscellaneous/scripts/alternative/test_alt_oss.openbb
+++ b/openbb_terminal/miscellaneous/scripts/alternative/test_alt_oss.openbb
@@ -1,8 +1,8 @@
 alternative
 oss
-rs openbb-finance/openbbterminal
-sh openbb-finance/openbbterminal
+rs ${repo=openbb-finance/openbbterminal}
+sh ${repo=openbb-finance/openbbterminal}
 tr
 rossidx
 rossidx --sortby Raised [$M] --reverse
-exit
\ No newline at end of file
+exit
diff --git a/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto.openbb b/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto.openbb
index df8776e2689..cf35901e779 100644
--- a/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto.openbb
+++ b/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto.openbb
@@ -1,37 +1,37 @@
 crypto
-load eth
+load ${crypto=eth}
 
 # # Find
-# find eth
-# find eth --key symbol
-# find eth --key id
-# find ethereum --key name
-# find eth --limit 5
-# find eth --skip 3 # Bug that only skips second similar coins but not first
-# find eth --source CoinGecko
-# find eth --source CoinPaprika
-# find eth --source Binance
-# find eth --source Coinbase
+# find ${crypto=eth}
+# find ${crypto=eth} --key symbol
+# find ${crypto=eth} --key id
+# find ${crypto_full=ethereum} --key name
+# find ${crypto=eth} --limit 5
+# find ${crypto=eth} --skip 3 # Bug that only skips second similar coins but not first
+# find ${crypto=eth} --source CoinGecko
+# find ${crypto=eth} --source CoinPaprika
+# find ${crypto=eth} --source Binance
+# find ${crypto=eth} --source Coinbase
 # # Bug with this source
-# # find eth --source Yahoofinance
+# # find ${crypto=eth} --source Yahoofinance
 
 # # Price
-# # price eth
-# price eth
+# # price ${crypto=eth}
+# price ${crypto=eth}
 
 # # Headlines
 # headlines
-# headlines ETH
+# headlines ${crypto=ETH}
 
 # # Candle
 # candle
 # candle --log
 
-# # PRT
-# prt --vs btc
-# # Bug where we need --vs when we don't actually need it
-# # prt --top 3
-# # Bug where --price doesn't seem to do anything
-# # prt --vs btc --price 1000
+# PRT
+# prt --vs ${crypto_vs=btc}
+# Bug where we need --vs when we don't actually need it
+# prt --top 3
+# Bug where --price doesn't seem to do anything
+# prt --vs ${crypto_vs=btc} --price 1000
 
 exit
diff --git a/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto_dd.openbb b/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto_dd.openbb
index 91d9a7b211a..5fb9d35d4d5 100644
--- a/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto_dd.openbb
+++ b/openbb_terminal/miscellaneous/scripts/cryptocurrency/test_crypto_dd.openbb
@@ -1,127 +1,70 @@
 crypto
-load eth
+load ${crypto=eth}
 dd
 
-# info
 info
 
-# ath
-ath --vs usd
+ath --vs ${currency_vs=usd}
+atl --vs ${currency_vs=usd}
 
-# atl
-atl --vs usd
-
-# web
 web
-
-# bc
 bc
-
-# pi
 pi
-
-# gov
 gov
-
-# basic
 basic
 
-# stats
 stats
-stats --vs GBP
+stats --vs ${currency_vs=GBP}
 
-# desc
 desc
-
-# market
 market
 
-# mkt
 mkt
 mkt --urls
-mkt --vs ETH --limit 3 --sort volume --urls
-mkt --vs BTC -s volume -r
+mkt --vs ${crypto_vs=ETH} --limit 3 --sort volume --urls
+mkt --vs {crypto_vs=ETH} --limit 3 --sort volume --urls
+mkt --vs {crypto_vs=BTC} -s volume -r
 
-# ex
 ex
 ex --limit 3 --sort adjusted_volume_24h_share
 
-# balance
 balance
 
-# oi
-
-# fundrate
-
-# liquidations
-
-# eb
-
-# trades
 trades
 trades -e aax
-trades --vs GBP
+trades --vs ${currency_vs=GBP}
 
-# ob
 ob
 ob -e aax
-ob --vs GBP
+ob --vs ${currency_vs=GBP}
 
-# ps
 ps
-ps --vs ETH
+ps --vs ${currency_vs=ETH}
 
-# mcapdom
 mcapdom
-
-# mt
 mt
-
-# funot
-
-# team
 tea