[BugFix] Exclude default values when model dumping (#6516)

* fix: exclude defaults when model dumping

* fix: rename method in system service

* fix: v3 to v4 keys mapping

* feat: settings json templates

* fix: unit test

* fix: unit test

12 files changed, 77 insertions, 69 deletions

diff --git a/openbb_platform/core/openbb_core/api/router/commands.py b/openbb_platform/core/openbb_core/api/router/commands.py
index ce1cab484a8..d342d2e5fd1 100644
--- a/openbb_platform/core/openbb_core/api/router/commands.py
+++ b/openbb_platform/core/openbb_core/api/router/commands.py
@@ -192,7 +192,7 @@ def build_api_wrapper(
         user_settings: UserSettings = UserSettings.model_validate(
             kwargs.pop(
                 "__authenticated_user_settings",
-                UserService.read_default_user_settings(),
+                UserService.read_from_file(),
             )
         )
         execute = partial(command_runner.run, path, user_settings)
diff --git a/openbb_platform/core/openbb_core/app/command_runner.py b/openbb_platform/core/openbb_core/app/command_runner.py
index edfe22436cc..ea0e513c499 100644
--- a/openbb_platform/core/openbb_core/app/command_runner.py
+++ b/openbb_platform/core/openbb_core/app/command_runner.py
@@ -410,7 +410,7 @@ class CommandRunner:
         """Initialize the command runner."""
         self._command_map = command_map or CommandMap()
         self._system_settings = system_settings or SystemService().system_settings
-        self._user_settings = user_settings or UserService.read_default_user_settings()
+        self._user_settings = user_settings or UserService.read_from_file()
 
     def init_logging_service(self) -> None:
         """Initialize the logging service."""
diff --git a/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py b/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py
index edf52f3111e..4a642f8d52b 100644
--- a/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py
+++ b/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py
@@ -2,7 +2,7 @@
 
 from typing import Any, Dict, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
 class HubUserSettings(BaseModel):
@@ -14,3 +14,9 @@ class HubUserSettings(BaseModel):
     # features_terminal_style: Dict[str, Union[str, Dict[str, str]]]
 
     model_config = ConfigDict(validate_assignment=True)
+
+    @field_validator("features_keys", mode="before", check_fields=False)
+    @classmethod
+    def to_lower(cls, d: dict) -> dict:
+        """Convert dict keys to lowercase."""
+        return {k.lower(): v for k, v in d.items()}
diff --git a/openbb_platform/core/openbb_core/app/model/system_settings.py b/openbb_platform/core/openbb_core/app/model/system_settings.py
index 470e265a9b4..0f985696243 100644
--- a/openbb_platform/core/openbb_core/app/model/system_settings.py
+++ b/openbb_platform/core/openbb_core/app/model/system_settings.py
@@ -66,9 +66,9 @@ class SystemSettings(Tagged):
         )
 
     @staticmethod
-    def create_empty_json(path: Path) -> None:
+    def create_json(path: Path, template: Optional[dict] = None) -> None:
         """Create an empty JSON file."""
-        path.write_text(json.dumps({}), encoding="utf-8")
+        path.write_text(json.dumps(obj=template or {}, indent=4), encoding="utf-8")
 
     # TODO: Figure out why this works only opposite to what the docs say
     # https://docs.pydantic.dev/latest/concepts/validators/#model-validators
@@ -82,9 +82,14 @@ class SystemSettings(Tagged):
         system_settings = Path(values.system_settings_path).resolve()
         obb_dir.mkdir(parents=True, exist_ok=True)
 
-        for path in [user_settings, system_settings]:
-            if not path.exists():
-                cls.create_empty_json(path)
+        if not user_settings.exists():
+            cls.create_json(
+                user_settings,
+                {"credentials": {}, "preferences": {}, "defaults": {"commands": {}}},
+            )
+
+        if not system_settings.exists():
+            cls.create_json(system_settings, {})
 
         return values
 
diff --git a/openbb_platform/core/openbb_core/app/service/hub_service.py b/openbb_platform/core/openbb_core/app/service/hub_service.py
index 3f5bbb8b7dc..52d92d793e2 100644
--- a/openbb_platform/core/openbb_core/app/service/hub_service.py
+++ b/openbb_platform/core/openbb_core/app/service/hub_service.py
@@ -22,14 +22,14 @@ class HubService:
     TIMEOUT = 10
     # Mapping of V3 keys to V4 keys for backward compatibility
     V3TOV4 = {
-        "API_KEY_ALPHAVANTAGE": "alpha_vantage_api_key",
-        "API_BIZTOC_TOKEN": "biztoc_api_key",
-        "API_FRED_KEY": "fred_api_key",
-        "API_KEY_FINANCIALMODELINGPREP": "fmp_api_key",
-        "API_INTRINIO_KEY": "intrinio_api_key",
-        "API_POLYGON_KEY": "polygon_api_key",
-        "API_KEY_QUANDL": "nasdaq_api_key",
-        "API_TRADIER_TOKEN": "tradier_api_key",
+        "api_key_alphavantage": "alpha_vantage_api_key",
+        "api_biztoc_token": "biztoc_api_key",
+        "api_fred_key": "fred_api_key",
+        "api_key_financialmodelingprep": "fmp_api_key",
+        "api_intrinio_key": "intrinio_api_key",
+        "api_polygon_key": "polygon_api_key",
+        "api_key_quandl": "nasdaq_api_key",
+        "api_tradier_token": "tradier_api_key",
     }
     V4TOV3 = {v: k for k, v in V3TOV4.items()}
 
@@ -218,7 +218,7 @@ class HubService:
         response = put(
             url=self._base_url + "/user",
             headers={"Authorization": authorization},
-            json=settings.model_dump(),
+            json=settings.model_dump(exclude_defaults=True),
             timeout=self.TIMEOUT,
         )
 
@@ -230,13 +230,13 @@ class HubService:
 
     def hub2platform(self, settings: HubUserSettings) -> Tuple[Credentials, Defaults]:
         """Convert Hub user settings to Platform models."""
-        if any(k in settings.features_keys for k in self.V3TOV4):
-            deprecated = {
-                k: v for k, v in self.V3TOV4.items() if k in settings.features_keys
-            }
+        deprecated = {
+            k: v for k, v in self.V3TOV4.items() if k in settings.features_keys
+        }
+        if deprecated:
             msg = ""
             for k, v in deprecated.items():
-                msg += f"\n'{k}' -> '{v.upper()}', "
+                msg += f"\n'{k.upper()}' -> '{v.upper()}', "
             msg = msg.strip(", ")
             warn(
                 message=f"\nDeprecated v3 credentials found.\n{msg}"
@@ -255,14 +255,18 @@ class HubService:
     ) -> HubUserSettings:
         """Convert Platform models to Hub user settings."""
         # Dump mode json ensures SecretStr values are serialized as strings
-        credentials = credentials.model_dump(mode="json", exclude_none=True)
+        credentials = credentials.model_dump(
+            mode="json", exclude_none=True, exclude_defaults=True
+        )
         settings = self._hub_user_settings or HubUserSettings()
         for v4_k, v in sorted(credentials.items()):
             v3_k = self.V4TOV3.get(v4_k, None)
             # If v3 key was in the hub already, we keep it
             k = v3_k if v3_k in settings.features_keys else v4_k
             settings.features_keys[k] = v
-        defaults_ = defaults.model_dump(mode="json", exclude_none=True)
+        defaults_ = defaults.model_dump(
+            mode="json", exclude_none=True, exclude_defaults=True
+        )
         settings.features_settings.update({"defaults": defaults_})
         return settings
 
diff --git a/openbb_platform/core/openbb_core/app/service/system_service.py b/openbb_platform/core/openbb_core/app/service/system_service.py
index 906ff6808ec..4d9f9c6feeb 100644
--- a/openbb_platform/core/openbb_core/app/service/system_service.py
+++ b/openbb_platform/core/openbb_core/app/service/system_service.py
@@ -31,7 +31,7 @@ class SystemService(metaclass=SingletonMeta):
         **kwargs,
     ):
         """Initialize system service."""
-        self._system_settings = self._read_default_system_settings(
+        self._system_settings = self._read_from_file(
             path=self.SYSTEM_SETTINGS_PATH, **kwargs
         )
 
@@ -46,9 +46,7 @@ class SystemService(metaclass=SingletonMeta):
         return hashed_input == existing_hash
 
     @classmethod
-    def _read_default_system_settings(
-        cls, path: Optional[Path] = None, **kwargs
-    ) -> SystemSettings:
+    def _read_from_file(cls, path: Optional[Path] = None, **kwargs) -> SystemSettings:
         """Read default system settings."""
         path = path or cls.SYSTEM_SETTINGS_PATH
 
@@ -77,7 +75,7 @@ class SystemService(metaclass=SingletonMeta):
         return system_settings
 
     @classmethod
-    def write_default_system_settings(
+    def write_to_file(
         cls,
         system_settings: SystemSettings,
         path: Optional[Path] = None,
@@ -86,7 +84,9 @@ class SystemService(metaclass=SingletonMeta):
         path = path or cls.SYSTEM_SETTINGS_PATH
 
         system_settings_json = system_settings.model_dump_json(
-            include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET, indent=4
+            indent=4,
+            include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET,
+            exclude_defaults=True,
         )
         with path.open(mode="w") as file:
             file.write(system_settings_json)
@@ -103,6 +103,6 @@ class SystemService(metaclass=SingletonMeta):
 
     def refresh_system_settings(self) -> SystemSettings:
         """Refresh system settings."""
-        self._system_settings = self._read_default_system_settings()
+        self._system_settings = self._read_from_file()
 
         return self._system_settings
diff --git a/openbb_platform/core/openbb_core/app/service/user_service.py b/openbb_platform/core/openbb_core/app/service/user_service.py
index 05c0c5fd8c7..5ed35acc448 100644
--- a/openbb_platform/core/openbb_core/app/service/user_service.py
+++ b/openbb_platform/core/openbb_core/app/service/user_service.py
@@ -21,13 +21,11 @@ class UserService(metaclass=SingletonMeta):
         default_user_settings: Optional[UserSettings] = None,
     ):
         """Initialize user service."""
-        self._default_user_settings = (
-            default_user_settings or self.read_default_user_settings()
-        )
+        self._default_user_settings = default_user_settings or self.read_from_file()
 
     @classmethod
-    def read_default_user_settings(cls, path: Optional[Path] = None) -> UserSettings:
-        """Read default user settings."""
+    def read_from_file(cls, path: Optional[Path] = None) -> UserSettings:
+        """Read user settings from json into UserSettings."""
         path = path or cls.USER_SETTINGS_PATH
 
         return (
@@ -37,16 +35,15 @@ class UserService(metaclass=SingletonMeta):
         )
 
     @classmethod
-    def write_default_user_settings(
+    def write_to_file(
         cls,
         user_settings: UserSettings,
         path: Optional[Path] = None,
     ) -> None:
-        """Write default user settings."""
+        """Write user settings to json."""
         path = path or cls.USER_SETTINGS_PATH
-
         user_settings_json = user_settings.model_dump_json(
-            include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, indent=4
+            indent=4, include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, exclude_defaults=True
         )
         path.write_text(user_settings_json, encoding="utf-8")
 
diff --git a/openbb_platform/core/openbb_core/app/static/account.py b/openbb_platform/core/openbb_core/app/static/account.py
index 8176c901052..d840b6b9fa3 100644
--- a/openbb_platform/core/openbb_core/app/static/account.py
+++ b/openbb_platform/core/openbb_core/app/static/account.py
@@ -156,9 +156,7 @@ class Account:  # noqa: D205, D400
             User settings: profile, credentials, preferences
         """
         if not self._hub_service:
-            UserService.write_default_user_settings(
-                self._base_app._command_runner.user_settings
-            )
+            UserService.write_to_file(self._base_app._command_runner.user_settings)
         else:
             self._hub_service.push(self._base_app._command_runner.user_settings)
 
@@ -181,9 +179,7 @@ class Account:  # noqa: D205, D400
             User settings: profile, credentials, preferences
         """
         if not self._hub_service:
-            self._base_app._command_runner.user_settings = (
-                UserService.read_default_user_settings()
-            )
+            self._base_app._command_runner.user_settings = UserService.read_from_file()
         else:
             incoming = self._hub_service.pull()
             self._base_app.user.profile = incoming.profile
@@ -216,9 +212,7 @@ class Account:  # noqa: D205, D400
         if session_file.exists():
             session_file.unlink()
 
-        self._base_app._command_runner.user_settings = (
-            UserService.read_default_user_settings()
-        )
+        self._base_app._command_runner.user_settings = UserService.read_from_file()
 
         if return_settings:
             return self._base_app._command_runner.user_settings
diff --git a/openbb_platform/core/tests/app/service/test_hub_service.py b/openbb_platform/core/tests/app/service/test_hub_service.py
index 903eef2059b..5499109e225 100644
--- a/openbb_platform/core/tests/app/service/test_hub_service.py
+++ b/openbb_platform/core/tests/app/service/test_hub_service.py
@@ -256,9 +256,9 @@ def test_hub2platform_v3_only():
     """Test hub2platform."""
     mock_user_settings = MagicMock(spec=HubUserSettings)
     mock_user_settings.features_keys = {
-        "API_KEY_FINANCIALMODELINGPREP": "abc",
-        "API_POLYGON_KEY": "def",
-        "API_FRED_KEY": "ghi",
+        "api_key_financialmodelingprep": "abc",
+        "api_polygon_key": "def",
+        "api_fred_key": "ghi",
     }
     mock_user_settings.features_settings = {}
 
@@ -273,10 +273,10 @@ def test_hub2platform_v3v4():
     """Test hub2platform."""
     mock_user_settings = MagicMock(spec=HubUserSettings)
     mock_user_settings.features_keys = {
-        "API_KEY_FINANCIALMODELINGPREP": "abc",
+        "api_key_financialmodelingprep": "abc",
         "fmp_api_key": "other_key",
-        "API_POLYGON_KEY": "def",
-        "API_FRED_KEY": "ghi",
+        "api_polygon_key": "def",
+        "api_fred_key": "ghi",
     }
     mock_user_settings.features_settings = {}
 
@@ -291,9 +291,9 @@ def test_platform2hub():
     """Test platform2hub."""
     mock_user_settings = MagicMock(spec=HubUserSettings)
     mock_user_settings.features_keys = {  # Received from Hub
-        "API_KEY_FINANCIALMODELINGPREP": "abc",
+        "api_key_financialmodelingprep": "abc",
         "fmp_api_key": "other_key",
-        "API_FRED_KEY": "ghi",
+        "api_fred_key": "ghi",
     }
     mock_user_settings.features_settings = {}
     mock_hub_service = HubService()
@@ -309,10 +309,10 @@ def test_platform2hub():
     user_settings = mock_hub_service.platform2hub(mock_credentials, mock_defaults)
 
     assert isinstance(user_settings, HubUserSettings)
-    assert user_settings.features_keys["API_KEY_FINANCIALMODELINGPREP"] == "fmp"
+    assert user_settings.features_keys["api_key_financialmodelingprep"] == "fmp"
     assert user_settings.features_keys["fmp_api_key"] == "other_key"
     assert user_settings.features_keys["polygon_api_key"] == "polygon"
-    assert user_settings.features_keys["API_FRED_KEY"] == "fred"
+    assert user_settings.features_keys["api_fred_key"] == "fred"
     assert user_settings.features_keys["benzinga_api_key"] == "benzinga"
     assert "some_api_key" not in user_settings.features_keys
     assert "defaults" in user_settings.features_settings
diff --git a/openbb_platform/core/tests/app/service/test_system_service.py b/openbb_platform/core/tests/app/service/test_system_service.py
index 5ecac85bbbe..a563a1a3771 100644
--- a/openbb_platform/core/tests/app/service/test_system_service.py
+++ b/openbb_platform/core/tests/app/service/test_system_service.py
@@ -17,17 +17,19 @@ def test_system_service_init(system_service):
     assert system_service
 
 
-def test_read_default_system_settings(system_service):
+def test_read_from_file(system_service):
     """Test read default system settings."""
-    system_settings = system_service._read_default_system_settings()
+    # pylint: disable=protected-access
+    system_settings = system_service._read_from_file()
 
     assert system_settings
 
 
-def test_write_default_system_settings(system_service):
+def test_write_to_file(system_service):
     """Test write default system settings."""
-    system_settings = system_service._read_default_system_settings()
-    system_service.write_default_system_settings(system_settings=system_settings)
+    # pylint: disable=protected-access
+    system_settings = system_service._read_from_file()
+    system_service.write_to_file(system_settings=system_settings)
 
     assert system_service
 
diff --git a/openbb_platform/core/tests/app/service/test_user_service.py b/openbb_platform/core/tests/app/service/test_user_service.py
index cf64ff98de2..f00ac2a347f 100644
--- a/openbb_platform/core/tests/app/service/test_user_service.py
+++ b/openbb_platform/core/tests/app/service/test_user_service.py
@@ -10,15 +10,15 @@ from openbb_core.app.service.user_service import (
 )
 
 
-def test_read_default_user_settings_file_exists():
+def test_read_from_file_file_exists():
     """Test read default user settings."""
-    result = UserService.read_default_user_settings(path=Path("some_path"))
+    result = UserService.read_from_file(path=Path("some_path"))
 
     assert result
     assert isinstance(result, UserSettings)
 
 
-def test_write_default_user_settings():
+def test_write_to_file():
     """Test write default user settings."""
     # Create a temporary file for this test
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -31,7 +31,7 @@ def test_write_default_user_settings():
     user_settings.defaults = {"language": "en"}  # type: ignore[assignment]
 
     # Write the user settings to the temporary file
-    UserService.write_default_user_settings(user_settings, temp_path)
+    UserService.write_to_file(user_settings, temp_path)
 
     # Read the file and verify its contents
     with open(temp_path, encoding="utf-8") as file:
diff --git a/openbb_platform/core/tests/app/static/test_container.py b/openbb_platform/core/tests/app/static/test_container.py
index 711a9b3bfe7..a19b1331b09 100644
--- a/openbb_platform/core/tests/app/static/test_container.py
+++ b/openbb_platform/core/tests/app/static/test_container.py
@@ -89,7 +89,7 @@ def test_container__check_credentials(container):
             OpenBBError,
             escape(
                 "Provider fallback failed."
-                "\n[Providers]\n  * 'x' -> not found\n  * 'y' -> not found\n  * 'z' -> not found"
+                "\n[Providers]\n  * 'x' -> not installed, please install openbb-x\n  * 'y' -> not installed, please install openbb-y\n  * 'z' -> not installed, please install openbb-z"
             ),
         ),
     ],