New path for presets (#2634)

* modify path for etf, options, insider

* fix small

* change path for stocks screener

* fix flake 8

* remove preset_path

* black

* fix

* fix tests

* fix tests

* fix flake8

* change paths_helper load

* put ns parser back

Co-authored-by: minhhoang1023 <40023817+minhhoang1023@users.noreply.github.com>

39 files changed, 23568 insertions, 185 deletions

diff --git a/openbb_terminal/config_terminal.py b/openbb_terminal/config_terminal.py
index cd5809da900..0e84dad04eb 100644
--- a/openbb_terminal/config_terminal.py
+++ b/openbb_terminal/config_terminal.py
@@ -6,9 +6,7 @@ from distutils.util import strtobool
 import dotenv
 
 # IMPORTATION INTERNAL
-from openbb_terminal.core.config import (  # pylint: disable=unused-import  # noqa
-    paths_helper,
-)
+
 from openbb_terminal.core.config.paths import USER_ENV_FILE, REPOSITORY_ENV_FILE
 from .helper_classes import TerminalStyle as _TerminalStyle
 
diff --git a/openbb_terminal/core/config/paths.py b/openbb_terminal/core/config/paths.py
index 8beeffa9567..622b7271123 100644
--- a/openbb_terminal/core/config/paths.py
+++ b/openbb_terminal/core/config/paths.py
@@ -30,3 +30,4 @@ CUSTOM_IMPORTS_DIRECTORY = USER_DATA_DIRECTORY / "custom_imports"
 PORTFOLIO_DATA_DIRECTORY = USER_DATA_DIRECTORY / "portfolio"
 ROUTINES_DIRECTORY = USER_DATA_DIRECTORY / "routines"
 DATA_SOURCES_DEFAULT_FILE = USER_DATA_DIRECTORY / "data_sources_default.json"
+PRESETS_DIRECTORY = USER_DATA_DIRECTORY / "presets"
diff --git a/openbb_terminal/core/config/paths_helper.py b/openbb_terminal/core/config/paths_helper.py
index 6c164d0b144..0625238996d 100644
--- a/openbb_terminal/core/config/paths_helper.py
+++ b/openbb_terminal/core/config/paths_helper.py
@@ -1,5 +1,4 @@
 # IMPORTATION STANDARD
-import os
 from typing import List
 
 from openbb_terminal.core.config.paths import (
@@ -10,10 +9,13 @@ from openbb_terminal.core.config.paths import (
     CUSTOM_IMPORTS_DIRECTORY,
     USER_EXPORTS_DIRECTORY,
     PORTFOLIO_DATA_DIRECTORY,
+    PRESETS_DIRECTORY,
     ROUTINES_DIRECTORY,
     DATA_SOURCES_DEFAULT_FILE,
 )
 
+# pylint: disable=W0603
+
 
 def create_paths(list_dirs: List):
     """
@@ -21,8 +23,10 @@ def create_paths(list_dirs: List):
     terminal if they don't exist
     """
     for dirs in list_dirs:
-        if not os.path.exists(dirs):
-            os.mkdir(dirs)
+        if not dirs.exists():
+            dirs.mkdir(
+                parents=True,
+            )
 
 
 def create_files(list_files: List):
@@ -50,8 +54,23 @@ dirs_list = [
     PORTFOLIO_DATA_DIRECTORY / "holdings",
     PORTFOLIO_DATA_DIRECTORY / "allocation",
     PORTFOLIO_DATA_DIRECTORY / "optimization",
+    PRESETS_DIRECTORY,
+    PRESETS_DIRECTORY / "stocks" / "options",
+    PRESETS_DIRECTORY / "stocks" / "screener",
+    PRESETS_DIRECTORY / "stocks" / "insider",
+    PRESETS_DIRECTORY / "etf" / "screener",
     ROUTINES_DIRECTORY,
 ]
 dirs_files = [USER_ENV_FILE, REPOSITORY_ENV_FILE, DATA_SOURCES_DEFAULT_FILE]
-create_paths(dirs_list)
-create_files(dirs_files)
+initialized = False
+
+
+def init_userdata():
+    """
+    Initializes the user data folder
+    """
+    global initialized
+    if not initialized:
+        create_paths(dirs_list)
+        create_files(dirs_files)
+        initialized = True
diff --git a/openbb_terminal/etf/screener/screener_controller.py b/openbb_terminal/etf/screener/screener_controller.py
index fada6ab70a2..c444c374e20 100644
--- a/openbb_terminal/etf/screener/screener_controller.py
+++ b/openbb_terminal/etf/screener/screener_controller.py
@@ -5,15 +5,15 @@ __docformat__ = "numpy"
 import argparse
 import configparser
 import logging
-import os
 from typing import List
 
 from prompt_toolkit.completion import NestedCompleter
 
 from openbb_terminal import feature_flags as obbff
+
 from openbb_terminal.decorators import log_start_end
 from openbb_terminal.etf import financedatabase_model, financedatabase_view
-from openbb_terminal.etf.screener import screener_view
+from openbb_terminal.etf.screener import screener_view, screener_model
 from openbb_terminal.helper_funcs import (
     EXPORT_ONLY_RAW_DATA_ALLOWED,
     check_positive,
@@ -24,8 +24,6 @@ from openbb_terminal.rich_config import console, MenuText
 
 logger = logging.getLogger(__name__)
 
-presets_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "presets/")
-
 
 class ScreenerController(BaseController):
     """Screener Controller class"""
@@ -37,10 +35,7 @@ class ScreenerController(BaseController):
         "sbc",
     ]
 
-    presets_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "presets/")
-    preset_choices = [
-        f.split(".")[0] for f in os.listdir(presets_path) if f.endswith(".ini")
-    ]
+    preset_choices = screener_model.get_preset_choices()
 
     sortby_screen_choices = [
         "Assets",
@@ -119,7 +114,7 @@ class ScreenerController(BaseController):
             if ns_parser.preset:
                 preset_filter = configparser.RawConfigParser()
                 preset_filter.optionxform = str  # type: ignore
-                preset_filter.read(presets_path + ns_parser.preset + ".ini")
+                preset_filter.read(self.preset_choices[ns_parser.preset])
 
                 headers = [
                     "Price",
@@ -152,7 +147,7 @@ class ScreenerController(BaseController):
                 console.print("\nPresets:")
                 for preset in self.preset_choices:
                     with open(
-                        presets_path + preset + ".ini",
+                        self.preset_choices[preset],
                         encoding="utf8",
                     ) as f:
                         description = ""
diff --git a/openbb_terminal/etf/screener/screener_model.py b/openbb_terminal/etf/screener/screener_model.py
index 6c48dc264ed..936ec444729 100644
--- a/openbb_terminal/etf/screener/screener_model.py
+++ b/openbb_terminal/etf/screener/screener_model.py
@@ -3,16 +3,43 @@ __docformat__ = "numpy"
 
 import configparser
 import logging
-import os
+from pathlib import Path
+from typing import Dict
 
 import pandas as pd
 
 from openbb_terminal.decorators import log_start_end
+from openbb_terminal.core.config.paths import PRESETS_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
 
 @log_start_end(log=logger)
+def get_preset_choices() -> Dict:
+    """
+    Return a dict containing keys as name of preset and
+    filepath as value
+    """
+
+    PRESETS_PATH = PRESETS_DIRECTORY / "etf" / "screener"
+    PRESETS_PATH_DEFAULT = Path(__file__).parent / "presets"
+    preset_choices = {
+        filepath.name: filepath
+        for filepath in PRESETS_PATH.iterdir()
+        if filepath.suffix == ".ini"
+    }
+    preset_choices.update(
+        {
+            filepath.name: filepath
+            for filepath in PRESETS_PATH_DEFAULT.iterdir()
+            if filepath.suffix == ".ini"
+        }
+    )
+
+    return preset_choices
+
+
+@log_start_end(log=logger)
 def etf_screener(preset: str):
     """
     Screens the etfs pulled from my repo (https://github.com/jmaslek/etf_scraper),
@@ -37,11 +64,8 @@ def etf_screener(preset: str):
     )
 
     cf = configparser.ConfigParser()
-    path = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)), "presets", preset + ".ini"
-    )
 
-    cf.read(path)
+    cf.read(get_preset_choices()[preset])
     cols = cf.sections()
 
     for col in cols:
diff --git a/openbb_terminal/feature_flags.py b/openbb_terminal/feature_flags.py
index 4dce35926b8..c554fb01b8b 100644
--- a/openbb_terminal/feature_flags.py
+++ b/openbb_terminal/feature_flags.py
@@ -10,6 +10,9 @@ import i18n
 
 # IMPORTATION INTERNAL
 from openbb_terminal.core.config.paths import USER_ENV_FILE, REPOSITORY_ENV_FILE, DATA_SOURCES_DEFAULT_FILE
+from openbb_terminal.core.config import paths_helper
+
+paths_helper.init_userdata()
 
 # pylint: disable=no-member,c-extension-no-member
 
diff --git a/openbb_terminal/stocks/insider/insider_controller.py b/openbb_terminal/stocks/insider/insider_controller.py
index 5607b2ae3b7..d173faee7bd 100644
--- a/openbb_terminal/stocks/insider/insider_controller.py
+++ b/openbb_terminal/stocks/insider/insider_controller.py
@@ -4,7 +4,6 @@ __docformat__ = "numpy"
 import argparse
 import configparser
 import logging
-import os
 from typing import List
 
 import pandas as pd
@@ -23,12 +22,11 @@ from openbb_terminal.stocks.insider import (
     businessinsider_view,
     finviz_view,
     openinsider_view,
+    openinsider_model,
 )
 
 logger = logging.getLogger(__name__)
 
-presets_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "presets/")
-
 # pylint: disable=,inconsistent-return-statements
 
 # TODO: HELP WANTED! This menu required some refactoring. Things that can be addressed:
@@ -69,11 +67,8 @@ class InsiderController(StockBaseController):
         "stats",
     ]
 
-    preset_choices = [
-        preset.split(".")[0]
-        for preset in os.listdir(presets_path)
-        if preset[-4:] == ".ini"
-    ]
+    preset_choices = openinsider_model.get_preset_choices()
+
     PATH = "/stocks/ins/"
 
     def __init__(
@@ -172,7 +167,7 @@ class InsiderController(StockBaseController):
             if ns_parser.preset:
                 preset_filter = configparser.RawConfigParser()
                 preset_filter.optionxform = str  # type: ignore
-                preset_filter.read(presets_path + ns_parser.preset + ".ini")
+                preset_filter.read(self.preset_choices[ns_parser.preset])
 
                 filters_headers = [
                     "General",
@@ -198,7 +193,7 @@ class InsiderController(StockBaseController):
             else:
                 for preset in self.preset_choices:
                     with open(
-                        presets_path + preset + ".ini",
+                        self.preset_choices[preset],
                         encoding="utf8",
                     ) as f:
                         description = ""
@@ -270,7 +265,7 @@ class InsiderController(StockBaseController):
         )
         if ns_parser:
             openinsider_view.print_insider_filter(
-                preset_loaded=self.preset,
+                preset=self.preset,
                 symbol="",
                 limit=ns_parser.limit,
                 links=ns_parser.urls,
@@ -311,7 +306,7 @@ class InsiderController(StockBaseController):
         if ns_parser:
             if self.ticker:
                 openinsider_view.print_insider_filter(
-                    preset_loaded="",
+                    preset="",
                     symbol=self.ticker,
                     limit=ns_parser.limit,
                     links=ns_parser.urls,
diff --git a/openbb_terminal/stocks/insider/openinsider_model.py b/openbb_terminal/stocks/insider/openinsider_model.py
index cd0911f1ae0..c494bfafd2e 100644
--- a/openbb_terminal/stocks/insider/openinsider_model.py
+++ b/openbb_terminal/stocks/insider/openinsider_model.py
@@ -1,6 +1,6 @@
 import configparser
 import logging
-import os
+from pathlib import Path
 import textwrap
 from datetime import datetime
 from typing import Dict, List
@@ -10,12 +10,11 @@ import requests
 from bs4 import BeautifulSoup
 
 from openbb_terminal.decorators import log_start_end
+from openbb_terminal.core.config.paths import PRESETS_DIRECTORY
 from openbb_terminal.rich_config import console
 
 logger = logging.getLogger(__name__)
 
-presets_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "presets/")
-
 
 # pylint: disable=too-many-branches,line-too-long,C0302
 # flake8: noqa
@@ -1104,6 +1103,31 @@ def check_open_insider_screener(
 
 
 @log_start_end(log=logger)
+def get_preset_choices() -> Dict:
+    """
+    Return a dict containing keys as name of preset and
+    filepath as value
+    """
+
+    PRESETS_PATH = PRESETS_DIRECTORY / "stocks" / "insider"
+    PRESETS_PATH_DEFAULT = Path(__file__).parent / "presets"
+    preset_choices = {
+        filepath.name: filepath
+        for filepath in PRESETS_PATH.iterdir()
+        if filepath.suffix == ".ini"
+    }
+    preset_choices.update(
+        {
+            filepath.name: filepath
+            for filepath in PRESETS_PATH_DEFAULT.iterdir()
+            if filepath.suffix == ".ini"
+        }
+    )
+
+    return preset_choices
+
+
+@log_start_end(log=logger)
 def get_open_insider_link(preset_loaded: str) -> str:
     """Get open insider link
 
@@ -1119,7 +1143,7 @@ def get_open_insider_link(preset_loaded: str) -> str:
     """
     preset = configparser.RawConfigParser()
     preset.optionxform = str  # type: ignore
-    preset.read(presets_path + preset_loaded + ".ini")
+    preset.read(get_preset_choices()[preset_loaded])
 
     d_general = dict(preset["General"])
     d_date = dict(preset["Date"])
diff --git a/openbb_terminal/stocks/insider/openinsider_view.py b/openbb_terminal/stocks/insider/openinsider_view.py
index a297e6b3e4b..291b96c08dd 100644
--- a/openbb_terminal/stocks/insider/openinsider_view.py
+++ b/openbb_terminal/stocks/insider/openinsider_view.py
@@ -141,7 +141,7 @@ def print_insider_data(type_insider: str = "lcb", limit: int = 10, export: str =
 
 @log_start_end(log=logger)
 def print_insider_filter(
-    preset_loaded: str,
+    preset: str,
     symbol: str,
     limit: int = 10,
     links: bool = False,
@@ -151,7 +151,7 @@ def print_insider_filter(
 
     Parameters
     ----------
-    preset_loaded : str
+    preset : str
         Loaded preset filter
     symbol : str
         Stock ticker symbol
@@ -165,7 +165,7 @@ def print_insider_filter(
     if symbol:
         link = f"http://openinsider.com/screener?s={symbol}"
     else:
-        link = get_open_insider_link(preset_loaded)
+        link = get_open_insider_link(preset)
 
     if not link:
         console.print("")
@@ -228,10 +228,10 @@ def print_insider_filter(
     )
 
     if export:
-        if preset_loaded:
-            cmd = "filter"
         if symbol:
             cmd = "lis"
+        else:
+            cmd = "filter"
 
         export_data(export, os.path.dirname(os.path.abspath(__file__)), cmd, df_insider)
 
diff --git a/openbb_terminal/stocks/options/options_controller.py b/openbb_terminal/stocks/options/options_controller.py
index 2398c7ab4fe..d188e8ec24d 100644
--- a/openbb_terminal/stocks/options/options_controller.py
+++ b/openbb_terminal/stocks/options/options_controller.py
@@ -3,7 +3,6 @@ __docformat__ = "numpy"
 
 import argparse
 import logging
-import os
 from datetime import datetime, timedelta
 from typing import Any, List
 
@@ -38,7 +37,11 @@ from openbb_terminal.stocks.options import (
 )
 from openbb_terminal.stocks.options.hedge import hedge_controller
 from openbb_terminal.stocks.options.pricing import pricing_controller
-from openbb_terminal.stocks.options.screen import screener_controller, syncretism_view
+from openbb_terminal.stocks.options.screen import (
+    screener_controller,
+    syncretism_view,
+    syncretism_model,
+)
 
 # pylint: disable=R1710,C0302,R0916
 
@@ -78,9 +81,7 @@ class OptionsController(BaseController):
         "hedge",
     ]
 
-    PRESET_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "presets/")
-
-    presets = [f.split(".")[0] for f in os.listdir(PRESET_PATH) if f.endswith(".ini")]
+    preset_choices = syncretism_model.get_preset_choices()
 
     grhist_greeks_choices = [
         "iv",
@@ -135,8 +136,8 @@ class OptionsController(BaseController):
             choices: dict = {c: {} for c in self.controller_choices}
             choices["unu"]["-s"] = {c: {} for c in self.unu_sortby_choices}
             choices["pcr"] = {c: {} for c