Refactor ca menu (#726)

* First wave of refactoring.  yfinance and marketwatch menus

* Finbrain commands refactored

* Refactor finviz commands

* Add custom ML model for running get

* Bug fixes + dim ca commands when no similar selected

* Add pylint disable

* move ml command into yfinance stuff.  fix typos

* Change historical/corr to have interested ticker first

* Pylint error

* Comment out fa test

13 files changed, 1849 insertions, 1417 deletions

diff --git a/gamestonk_terminal/stocks/comparison_analysis/ca_controller.py b/gamestonk_terminal/stocks/comparison_analysis/ca_controller.py
index d2a64f04806..4b2d58199ae 100644
--- a/gamestonk_terminal/stocks/comparison_analysis/ca_controller.py
+++ b/gamestonk_terminal/stocks/comparison_analysis/ca_controller.py
@@ -1,26 +1,35 @@
 """Comparison Analysis Controller Module"""
 __docformat__ = "numpy"
-
+# pylint:disable=too-many-lines
 import argparse
 import os
 import random
 from typing import List
-from datetime import datetime
-import requests
+
+from colorama import Style
 import pandas as pd
 from matplotlib import pyplot as plt
-from finvizfinance.screener.overview import Overview
 from prompt_toolkit.completion import NestedCompleter
 
 from gamestonk_terminal import feature_flags as gtff
-from gamestonk_terminal import config_terminal as cfg
-from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn
-from gamestonk_terminal.stocks.comparison_analysis import yahoo_finance_view
-from gamestonk_terminal.stocks.comparison_analysis import market_watch_view
-from gamestonk_terminal.stocks.comparison_analysis import finbrain_view
-from gamestonk_terminal.stocks.comparison_analysis import finviz_compare_view
-from gamestonk_terminal.portfolio.portfolio_optimization import po_controller
+from gamestonk_terminal.helper_funcs import (
+    check_non_negative,
+    get_flair,
+    parse_known_args_and_warn,
+)
 from gamestonk_terminal.menu import session
+from gamestonk_terminal.portfolio.portfolio_optimization import po_controller
+from gamestonk_terminal.stocks.comparison_analysis import (
+    finbrain_view,
+    finnhub_model,
+    finviz_compare_model,
+    finviz_compare_view,
+    marketwatch_view,
+    polygon_model,
+    yahoo_finance_view,
+    yahoo_finance_model,
+)
+from gamestonk_terminal.stocks.stocks_helper import load
 
 # pylint: disable=E1121
 
@@ -29,14 +38,15 @@ class ComparisonAnalysisController:
     """Comparison Analysis Controller class"""
 
     # Command choices
-    CHOICES = [
-        "?",
-        "cls",
-        "help",
-        "q",
-        "quit",
-        "get",
+    CHOICES = ["?", "cls", "help", "q", "quit"]
+
+    CHOICES_COMMANDS = [
+        "load",
+        "getpoly",
+        "getfinnhub",
+        "getfinviz",
         "select",
+        "add",
         "historical",
         "hcorr",
         "income",
@@ -50,13 +60,15 @@ class ComparisonAnalysisController:
         "ownership",
         "performance",
         "technical",
-        "po",
+        "tsne",
     ]
+    CHOICES_MENUS = ["po"]
+    CHOICES += CHOICES_COMMANDS + CHOICES_MENUS
 
     def __init__(
         self,
         ticker: str,
-        start: datetime,
+        start: str,
         interval: str,
         stock: pd.DataFrame,
     ):
@@ -66,7 +78,7 @@ class ComparisonAnalysisController:
         ----------
         ticker : str
             Stock ticker
-        start : datetime
+        start : str
             Start time
         interval : str
             Time interval
@@ -89,135 +101,186 @@ class ComparisonAnalysisController:
 
     def print_help(self):
         """Print help"""
-        print(
-            "https://github.com/GamestonkTerminal/GamestonkTerminal/tree/main/gamestonk_terminal"
-            "/stocks/comparison_analysis"
-        )
-
+        all_loaded = bool(not self.ticker or not self.similar)
         s_intraday = (f"Intraday {self.interval}", "Daily")[self.interval == "1440min"]
-
         if self.start:
-            print(
-                f"\n{s_intraday} Stock: {self.ticker} (from {self.start.strftime('%Y-%m-%d')})"
-            )
+            stock_str = f"{s_intraday} Stock: {self.ticker} (from {self.start.strftime('%Y-%m-%d')})"
         else:
-            print(f"\n{s_intraday} Stock: {self.ticker}")
-
-        if self.similar:
-            print(f"[{self.user}] Similar Companies: {', '.join(self.similar)}")
-
-        print("\nComparison Analysis Mode:")
-        print("   cls           clear screen")
-        print("   ?/help        show this menu again")
-        print("   q             quit this menu, and shows back to main menu")
-        print("   quit          quit to abandon program")
-        print("")
-        print("   get           get similar companies")
-        print("   select        select similar companies")
-        print("")
-        print("   historical    historical price data comparison")
-        print("   hcorr         historical price correlation")
-        print("   income        income financials comparison")
-        print("   balance       balance financials comparison")
-        print("   cashflow      cashflow comparison")
-        print("   sentiment     sentiment analysis comparison")
-        print("   scorr         sentiment correlation")
-        print("")
-        print("   overview      brief overview comparison")
-        print("   valuation     brief valuation comparison")
-        print("   financial     brief financial comparison")
-        print("   ownership     brief ownership comparison")
-        print("   performance   brief performance comparison")
-        print("   technical     brief technical comparison")
-        print("")
-
-        if self.similar:
-            print(">  po          portfolio optimization for selected tickers")
-            print("")
-        return
+            stock_str = f"{s_intraday} Stock: {self.ticker}"
+        help_str = f"""
+https://github.com/GamestonkTerminal/GamestonkTerminal/tree/main/gamestonk_terminal/stocks/comparison_analysis
+
+{stock_str}
+Similar Companies: {', '.join(self.similar) or None}
+
+Comparison Analysis:
+    cls           clear screen
+    ?/help        show this menu again
+    q             quit this menu, and shows back to main menu
+    quit          quit to abandon program
+
+    load          load new base ticker
+    add           add more companies to current selected (max 10 total)
+    select        reset and select similar companies
+
+Get Similar:
+    tsne          run TSNE on all SP500 stocks and returns 10 closest tickers
+    getpoly       get similar stocks from polygon API
+    getfinnhub    get similar stocks from finnhub API
+    getfinviz     get similar stocks from finviz API
+{Style.DIM if all_loaded else Style.NORMAL}Yahoo Finance:
+    historical    historical price data comparison
+    hcorr         historical price correlation {Style.RESET_ALL}
+Market Watch:
+    income        income financials comparison
+    balance       balance financials comparison
+    cashflow      cashflow comparison
+Finbrain:
+    sentiment     sentiment analysis comparison {Style.DIM if all_loaded else Style.NORMAL}
+    scorr         sentiment correlation{Style.RESET_ALL}
+Finviz:
+    overview      brief overview comparison
+    valuation     brief valuation comparison
+    financial     brief financial comparison
+    ownership     brief ownership comparison
+    performance   brief performance comparison
+    technical     brief technical comparison
+
+>   po            portfolio optimization for selected tickers
+        """
+        print(help_str)
 
-    def get_similar_companies(self, other_args: List[str]):
-        """Get similar companies. [Source: Polygon API]
+    def call_load(self, other_args: List[str]):
+        """Process load command"""
+        self.ticker, self.start, self.interval, self.stock = load(
+            other_args, self.ticker, self.start, self.interval, self.stock
+        )
+        if "." in self.ticker:
+            self.ticker = self.ticker.split(".")[0]
 
-        Parameters
-        ----------
-        other_args : List[str]
-            argparse other args
-        """
+    def call_tsne(self, other_args: List[str]):
+        """Process tsne command"""
         parser = argparse.ArgumentParser(
             add_help=False,
             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-            prog="get",
-            description="""Get similar companies to compare with.""",
+            prog="tsne",
+            description="""Get similar companies to compare with using sklearn TSNE.""",
         )
         parser.add_argument(
-            "-s",
-            "--source",
-            action="store",
-            default="finviz",
-            dest="source",
-            choices=["polygon", "finnhub", "finviz"],
-            help="source that provides similar companies",
-        )
-
-        # If source is finviz the user may want to get
-        # similar companies based on Industry and Sector only, and not
-        # on the fact that they are based on the same country
-        if "finviz" in other_args or not other_args:
-            parser.add_argument(
-                "--nocountry",
-                action="store_true",
-                default=False,
-                dest="b_no_country",
-                help="Similar stocks from finviz using only Industry and Sector.",
-            )
+            "-l",
+            "--learnrate",
+            default=200,
+            dest="lr",
+            type=check_non_negative,
+            help="TSNE Learning rate.  Typical values are between 50 and 200",
+        )
+        parser.add_argument(
+            "-p", "--no_plot", action="store_true", default=False, dest="no_plot"
+        )
 
         try:
-            if other_args:
-                if "-" not in other_args[0]:
-                    other_args.insert(0, "-s")
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+            self.similar = yahoo_finance_model.get_sp500_comps_tsne(
+                self.ticker, lr=ns_parser.lr, no_plot=ns_parser.no_plot
+            )
+            print(f"[ML] Similar Companies: {', '.join(self.similar)}", "\n")
+        except Exception as e:
+            print(e, "\n")
 
+    def call_getfinviz(self, other_args: List[str]):
+        """Process getfinviz command"""
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="getfinviz",
+            description="""Get similar companies from finviz to compare with.""",
+        )
+        parser.add_argument(
+            "--nocountry",
+            action="store_true",
+            default=False,
+            dest="b_no_country",
+            help="Similar stocks from finviz using only Industry and Sector.",
+        )
+        try:
             ns_parser = parse_known_args_and_warn(parser, other_args)
             if not ns_parser:
                 return
+            if ns_parser.b_no_country:
+                compare_list = ["Sector", "Industry"]
+            else:
+                compare_list = ["Sector", "Industry", "Country"]
 
-            if ns_parser.source == "polygon":
-                result = requests.get(
-                    f"https://api.polygon.io/v1/meta/symbols/{self.ticker.upper()}/company?&apiKey={cfg.API_POLYGON_KEY}"
-                )
+            self.similar, self.user = finviz_compare_model.get_similar_companies(
+                self.ticker, compare_list
+            )
 
-                if result.status_code == 200:
-                    self.similar = result.json()["similar"]
-                    self.user = "Polygon"
-                else:
-                    print(result.json()["error"])
+            if self.ticker.upper() in self.similar:
+                self.similar.remove(self.ticker.upper())
 
-            elif ns_parser.source == "finnhub":
-                result = requests.get(
-                    f"https://finnhub.io/api/v1/stock/peers?symbol={self.ticker}&token={cfg.API_FINNHUB_KEY}"
+            if len(self.similar) > 10:
+                random.shuffle(self.similar)
+                self.similar = sorted(self.similar[:10])
+                print(
+                    "The limit of stocks to compare with are 10. Hence, 10 random similar stocks will be displayed.\n",
                 )
 
-                if result.status_code == 200:
-                    d_peers = result.json()
+            if self.similar:
+                print(
+                    f"[{self.user}] Similar Companies: {', '.join(self.similar)}", "\n"
+                )
+        except Exception as e:
+            print(e, "\n")
 
-                    if d_peers:
-                        self.similar = d_peers
-                        self.user = "Finnhub"
-                    else:
-                        print("Similar companies not found.")
+    def call_getpoly(self, other_args: List[str]):
+        """Process get command"""
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="getpoly",
+            description="""Get similar companies from polygon to compare with.""",
+        )
 
-            else:
-                if ns_parser.b_no_country:
-                    compare_list = ["Sector", "Industry"]
-                else:
-                    compare_list = ["Sector", "Industry", "Country"]
-
-                self.similar = (
-                    Overview()
-                    .compare(self.ticker, compare_list, verbose=0)["Ticker"]
-                    .to_list()
+        try:
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+            self.similar, self.user = polygon_model.get_similar_companies(self.ticker)
+
+            if self.ticker.upper() in self.similar:
+                self.similar.remove(self.ticker.upper())
+
+            if len(self.similar) > 10:
+                random.shuffle(self.similar)
+                self.similar = sorted(self.similar[:10])
+                print(
+                    "The limit of stocks to compare with are 10. Hence, 10 random similar stocks will be displayed.\n",
+                )
+
+            if self.similar:
+                print(
+                    f"[{self.user}] Similar Companies: {', '.join(self.similar)}", "\n"
                 )
-                self.user = "Finviz"
+
+        except Exception as e:
+            print(e, "\n")
+
+    def call_getfinnhub(self, other_args: List[str]):
+        """Process get command"""
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="getfinnhub",
+            description="""Get similar companies from finnhubto compare with.""",
+        )
+        try:
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+
+            self.similar, self.user = finnhub_model.get_similar_companies(self.ticker)
 
             if self.ticker.upper() in self.similar:
                 self.similar.remove(self.ticker.upper())
@@ -230,20 +293,49 @@ class ComparisonAnalysisController:
                 )
 
             if self.similar:
-                print(f"[{self.user}] Similar Companies: {', '.join(self.similar)}")
-            print("")
+                print(
+                    f"[{self.user}] Similar Companies: {', '.join(self.similar)}", "\n"
+                )
 
         except Exception as e:
             print(e, "\n")
 
-    def select_similar_companies(self, other_args: List[str]):
-        """Select similar companies, e.g. NIO,XPEV,LI
+    def call_add(self, other_args: List[str]):
+        """Process add command"""
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="add",
+            description="""Add similar companies to compare with.""",
+        )
+        parser.add_argument(
+            "-s",
+            "--similar",
+            dest="l_similar",
+            type=lambda s: [str(item).upper() for item in s.split(",")],
+            default=[],
+            help="similar companies to compare with.",
+        )
+
+        try:
+            # For the case where a user uses: 'add NIO,XPEV,LI'
+            if other_args and "-" not in other_args[0]:
+                other_args.insert(0, "-s")
 
-        Parameters
-        ----------
-        other_args : List[str]
-            argparse other args
-        """
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+            # Add sets to avoid duplicates
+            self.similar = list(set(self.similar + ns_parser.l_similar))
+            if len(self.similar) > 10:
+                self.similar = list(random.sample(set(self.similar), 10))
+            self.user = "Custom"
+            print(f"[{self.user}] Similar Companies: {', '.join(self.similar)}", "\n")
+        except Exception as e:
+            print(e, "\n")
+
+    def call_select(self, other_args: List[str]):
+        """Process select command"""
         parser = argparse.ArgumentParser(
             add_help=False,
             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -261,17 +353,16 @@ class ComparisonAnalysisController:
 
         try:
             # For the case where a user uses: 'select NIO,XPEV,LI'
-            if other_args:
-                if "-" not in other_args[0]:
-                    other_args.insert(0, "-s")
+            if other_args and "-" not in other_args[0]:
+                other_args.insert(0, "-s")
 
             ns_parser = parse_known_args_and_warn(parser, other_args)
             if not ns_parser:
                 return
 
             self.similar = ns_parser.l_similar
-            self.user = "User"
-            print("")
+            self.user = "Custom"
+            print(f"[{self.user}] Similar Companies: {', '.join(self.similar)}", "\n")
 
         except Exception as e:
             print(e, "\n")
@@ -320,85 +411,564 @@ class ComparisonAnalysisController:
         """Process Quit command - quit the program"""
         return True
 
-    def call_get(self, other_args: List[str]):
-        """Process get command"""
-        self.get_similar_companies(other_args)
-
-    def call_select(self, other_args: List[str]):
-        """Process select command"""
-        self.select_similar_companies(other_args)
-
     def call_historical(self, other_args: List[str]):
         """Process historical command"""
-        yahoo_finance_view.historical(
-            other_args, self.stock, self.ticker, self.start, self.interval, self.similar
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="historical",
+            description="""Historical price comparison between similar companies.
+            """,
+        )
+        parser.add_argument(
+            "-t",
+            "--type",
+            action="store",
+            dest="type_candle",
+            type=str,
+            choices=["o", "h", "l", "c", "a"],
+            default="a",  # in case it's adjusted close
+            help="Candle data to use: o-open, h-high, l-low, c-close, a-adjusted close.",
+        )
+        parser.add_argument(
+            "-s",
+            "--no-scale",
+            action="store_false",
+            dest="no_scale",
+            default=True,
+            help="Flag to not put all prices on same 0-1 scale",
+        )
+        parser.add_argument(
+            "--export",
+            choices=["csv", "json", "xlsx"],
+            default="",
+            type=str,
+            dest="export",
+            help="Export dataframe data to csv,json,xlsx file",
         )
 
+        try:
+            if self.interval != "1440min":
+                print(
+                    "Intraday historical data analysis comparison is not yet available."
+                )
+                # Alpha Vantage only supports 5 calls per minute, we need another API to get intraday data
+            else:
+                ns_parser = parse_known_args_and_warn(parser, other_args)
+                if not ns_parser:
+                    return
+
+                if not self.similar or not self.ticker:
+                    print(
+                        "Please make sure there are both a loaded ticker and similar tickers selected. \n"
+                    )
+                    return
+
+                yahoo_finance_view.display_historical(
+                    ticker=self.ticker,
+                    similar_tickers=self.similar,
+                    start=self.start,
+                    candle_type=ns_parser.type_candle,
+                    normalize=ns_parser.no_scale,
+                    export=ns_parser.export,
+                )
+
+        except Exception as e:
+            print(e, "\n")
+
     def call_hcorr(self, other_args: List[str]):
         """Process historical correlation command"""
-        yahoo_finance_view.correlation(
-            other_args, self.stock, self.ticker, self.start, self.interval, self.similar
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="corr",
+            description=""" Correlation heatmap based on historical price comparison between similar
+            companies.
+            """,
+        )
+        parser.add_argument(
+            "-t",
+            "--type",
+            action="store",
+            dest="type_candle",
+            type=str,
+            choices=["o", "h", "l", "c", "a"],
+            default="a",  # in case it's adjusted close
+            help="Candle data to use: o-open, h-high, l-low, c-close, a-adjusted close.",
         )
 
+        try:
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+
+            if not self.similar or not self.ticker:
+                print(
+                    "Please make sure there are both a loaded ticker and similar tickers selected. \n"
+                )
+                return
+
+            yahoo_finance_view.display_correlation(
+                ticker=self.ticker,
+                similar_tickers=self.similar,
+                start=self.start,
+                candle_type=ns_parser.type_candle,
+            )
+        except Exception as e:
+            print(e, "\n")
+
     def call_income(self, other_args: List[str]):
         """Process income command"""
-        market_watch_view.compare_income(other_args, self.ticker, self.similar)
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="income",
+            description="""
+                Prints either yearly or quarterly income statement the company, and compares
+                it against similar companies.
+            """,
+        )
+        parser.add_argument(
+            "-q",
+            "--quarter",
+            action="store_true",
+            default=False,
+            dest="b_quarter",
+            help="Quarter financial data flag.",
+        )
+        parser.add_argument(
+            "-t",
+            "--timeframe",
+            dest="s_timeframe",
+            type=str,
+            default=None,
+            help="Specify yearly/quarterly timeframe. Default is last.",
+        )
+        parser.add_argument(
+            "--export",
+            choices=["csv", "json", "xlsx"],
+            default="",
+            type=str,
+            dest="export",
+            help="Export dataframe data to csv,json,xlsx file",
+        )
+
+        try:
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+
+            marketwatch_view.display_income_comparison(
+                ticker=self.ticker,
+                similar=self.similar,
+                timeframe=ns_parser.s_timeframe,
+                quarter=ns_parser.b_quarter,
+            )
+        except Exception as e:
+            print(e, "\n")
 
     def call_balance(self, other_args: List[str]):
         """Process balance command"""
-        market_watch_view.compare_balance(other_args, self.ticker, self.similar)
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="balance",
+            description="""
+                Prints either yearly or quarterly balance statement the company, and compares
+                it against similar companies..
+            """,
+        )
+        parser.add_argument(
+            "-q",
+            "--quarter",
+            action="store_true",
+            default=False,
+            dest="b_quarter",
+            help="Quarter financial data flag.",
+        )
+
+        parser.add_argument(
+            "-t",
+            "--timeframe",
+            dest="s_timeframe",
+            type=str,
+            default=None,
+            help="Specify yearly/quarterly timeframe. Default is last.",
+        )
+        parser.add_argument(
+            "--export",
+            choices=["csv", "json", "xlsx"],
+            default="",
+            type=str,
+            dest="export",
+            help="Export dataframe data to csv,json,xlsx file",
+        )
+
+        try:
+            ns_parser = parse_known_args_and_warn(parser, other_args)
+            if not ns_parser:
+                return
+
+            marketwatch_view.display_balance_comparison(
+                ticker=self.ticker,
+                similar=self.similar,
+                timeframe=ns_parser.s_timeframe,
+                quarter=ns_parser.b_quarter,
+            )
+
+        except Exception as e:
+            print(e, "\n")
 
     def call_cashflow(self, other_args: List[str]):
         """Process cashflow command"""
-        market_watch_view.compare_cashflow(other_args, self.ticker, self.similar)
+        parser = argparse.ArgumentParser(
+            add_help=False,