diff options
author | montezdesousa <79287829+montezdesousa@users.noreply.github.com> | 2024-06-20 19:54:16 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-06-20 18:54:16 +0000 |
commit | 604b69ccd8d16972a148b5c6c2f87d6fb6868358 (patch) | |
tree | edea928b988dfe1ec50017dd87ddbc1002253c7d | |
parent | ea61e6d5e7812739ea091301cd83ffcb9ef05189 (diff) |
[BugFix] Exclude default values when model dumping (#6516)
* fix: exclude defaults when model dumping
* fix: rename method in system service
* fix: v3 to v4 keys mapping
* feat: settings json templates
* fix: unit test
* fix: unit test
12 files changed, 77 insertions, 69 deletions
diff --git a/openbb_platform/core/openbb_core/api/router/commands.py b/openbb_platform/core/openbb_core/api/router/commands.py index ce1cab484a8..d342d2e5fd1 100644 --- a/openbb_platform/core/openbb_core/api/router/commands.py +++ b/openbb_platform/core/openbb_core/api/router/commands.py @@ -192,7 +192,7 @@ def build_api_wrapper( user_settings: UserSettings = UserSettings.model_validate( kwargs.pop( "__authenticated_user_settings", - UserService.read_default_user_settings(), + UserService.read_from_file(), ) ) execute = partial(command_runner.run, path, user_settings) diff --git a/openbb_platform/core/openbb_core/app/command_runner.py b/openbb_platform/core/openbb_core/app/command_runner.py index edfe22436cc..ea0e513c499 100644 --- a/openbb_platform/core/openbb_core/app/command_runner.py +++ b/openbb_platform/core/openbb_core/app/command_runner.py @@ -410,7 +410,7 @@ class CommandRunner: """Initialize the command runner.""" self._command_map = command_map or CommandMap() self._system_settings = system_settings or SystemService().system_settings - self._user_settings = user_settings or UserService.read_default_user_settings() + self._user_settings = user_settings or UserService.read_from_file() def init_logging_service(self) -> None: """Initialize the logging service.""" diff --git a/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py b/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py index edf52f3111e..4a642f8d52b 100644 --- a/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py +++ b/openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator class HubUserSettings(BaseModel): @@ -14,3 +14,9 @@ class HubUserSettings(BaseModel): # features_terminal_style: Dict[str, Union[str, Dict[str, str]]] model_config = ConfigDict(validate_assignment=True) + + @field_validator("features_keys", mode="before", check_fields=False) + @classmethod + def to_lower(cls, d: dict) -> dict: + """Convert dict keys to lowercase.""" + return {k.lower(): v for k, v in d.items()} diff --git a/openbb_platform/core/openbb_core/app/model/system_settings.py b/openbb_platform/core/openbb_core/app/model/system_settings.py index 470e265a9b4..0f985696243 100644 --- a/openbb_platform/core/openbb_core/app/model/system_settings.py +++ b/openbb_platform/core/openbb_core/app/model/system_settings.py @@ -66,9 +66,9 @@ class SystemSettings(Tagged): ) @staticmethod - def create_empty_json(path: Path) -> None: + def create_json(path: Path, template: Optional[dict] = None) -> None: """Create an empty JSON file.""" - path.write_text(json.dumps({}), encoding="utf-8") + path.write_text(json.dumps(obj=template or {}, indent=4), encoding="utf-8") # TODO: Figure out why this works only opposite to what the docs say # https://docs.pydantic.dev/latest/concepts/validators/#model-validators @@ -82,9 +82,14 @@ class SystemSettings(Tagged): system_settings = Path(values.system_settings_path).resolve() obb_dir.mkdir(parents=True, exist_ok=True) - for path in [user_settings, system_settings]: - if not path.exists(): - cls.create_empty_json(path) + if not user_settings.exists(): + cls.create_json( + user_settings, + {"credentials": {}, "preferences": {}, "defaults": {"commands": {}}}, + ) + + if not system_settings.exists(): + cls.create_json(system_settings, {}) return values diff --git a/openbb_platform/core/openbb_core/app/service/hub_service.py b/openbb_platform/core/openbb_core/app/service/hub_service.py index 3f5bbb8b7dc..52d92d793e2 100644 --- a/openbb_platform/core/openbb_core/app/service/hub_service.py +++ b/openbb_platform/core/openbb_core/app/service/hub_service.py @@ -22,14 +22,14 @@ class HubService: TIMEOUT = 10 # Mapping of V3 keys to V4 keys for backward compatibility V3TOV4 = { - "API_KEY_ALPHAVANTAGE": "alpha_vantage_api_key", - "API_BIZTOC_TOKEN": "biztoc_api_key", - "API_FRED_KEY": "fred_api_key", - "API_KEY_FINANCIALMODELINGPREP": "fmp_api_key", - "API_INTRINIO_KEY": "intrinio_api_key", - "API_POLYGON_KEY": "polygon_api_key", - "API_KEY_QUANDL": "nasdaq_api_key", - "API_TRADIER_TOKEN": "tradier_api_key", + "api_key_alphavantage": "alpha_vantage_api_key", + "api_biztoc_token": "biztoc_api_key", + "api_fred_key": "fred_api_key", + "api_key_financialmodelingprep": "fmp_api_key", + "api_intrinio_key": "intrinio_api_key", + "api_polygon_key": "polygon_api_key", + "api_key_quandl": "nasdaq_api_key", + "api_tradier_token": "tradier_api_key", } V4TOV3 = {v: k for k, v in V3TOV4.items()} @@ -218,7 +218,7 @@ class HubService: response = put( url=self._base_url + "/user", headers={"Authorization": authorization}, - json=settings.model_dump(), + json=settings.model_dump(exclude_defaults=True), timeout=self.TIMEOUT, ) @@ -230,13 +230,13 @@ class HubService: def hub2platform(self, settings: HubUserSettings) -> Tuple[Credentials, Defaults]: """Convert Hub user settings to Platform models.""" - if any(k in settings.features_keys for k in self.V3TOV4): - deprecated = { - k: v for k, v in self.V3TOV4.items() if k in settings.features_keys - } + deprecated = { + k: v for k, v in self.V3TOV4.items() if k in settings.features_keys + } + if deprecated: msg = "" for k, v in deprecated.items(): - msg += f"\n'{k}' -> '{v.upper()}', " + msg += f"\n'{k.upper()}' -> '{v.upper()}', " msg = msg.strip(", ") warn( message=f"\nDeprecated v3 credentials found.\n{msg}" @@ -255,14 +255,18 @@ class HubService: ) -> HubUserSettings: """Convert Platform models to Hub user settings.""" # Dump mode json ensures SecretStr values are serialized as strings - credentials = credentials.model_dump(mode="json", exclude_none=True) + credentials = credentials.model_dump( + mode="json", exclude_none=True, exclude_defaults=True + ) settings = self._hub_user_settings or HubUserSettings() for v4_k, v in sorted(credentials.items()): v3_k = self.V4TOV3.get(v4_k, None) # If v3 key was in the hub already, we keep it k = v3_k if v3_k in settings.features_keys else v4_k settings.features_keys[k] = v - defaults_ = defaults.model_dump(mode="json", exclude_none=True) + defaults_ = defaults.model_dump( + mode="json", exclude_none=True, exclude_defaults=True + ) settings.features_settings.update({"defaults": defaults_}) return settings diff --git a/openbb_platform/core/openbb_core/app/service/system_service.py b/openbb_platform/core/openbb_core/app/service/system_service.py index 906ff6808ec..4d9f9c6feeb 100644 --- a/openbb_platform/core/openbb_core/app/service/system_service.py +++ b/openbb_platform/core/openbb_core/app/service/system_service.py @@ -31,7 +31,7 @@ class SystemService(metaclass=SingletonMeta): **kwargs, ): """Initialize system service.""" - self._system_settings = self._read_default_system_settings( + self._system_settings = self._read_from_file( path=self.SYSTEM_SETTINGS_PATH, **kwargs ) @@ -46,9 +46,7 @@ class SystemService(metaclass=SingletonMeta): return hashed_input == existing_hash @classmethod - def _read_default_system_settings( - cls, path: Optional[Path] = None, **kwargs - ) -> SystemSettings: + def _read_from_file(cls, path: Optional[Path] = None, **kwargs) -> SystemSettings: """Read default system settings.""" path = path or cls.SYSTEM_SETTINGS_PATH @@ -77,7 +75,7 @@ class SystemService(metaclass=SingletonMeta): return system_settings @classmethod - def write_default_system_settings( + def write_to_file( cls, system_settings: SystemSettings, path: Optional[Path] = None, @@ -86,7 +84,9 @@ class SystemService(metaclass=SingletonMeta): path = path or cls.SYSTEM_SETTINGS_PATH system_settings_json = system_settings.model_dump_json( - include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET, indent=4 + indent=4, + include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET, + exclude_defaults=True, ) with path.open(mode="w") as file: file.write(system_settings_json) @@ -103,6 +103,6 @@ class SystemService(metaclass=SingletonMeta): def refresh_system_settings(self) -> SystemSettings: """Refresh system settings.""" - self._system_settings = self._read_default_system_settings() + self._system_settings = self._read_from_file() return self._system_settings diff --git a/openbb_platform/core/openbb_core/app/service/user_service.py b/openbb_platform/core/openbb_core/app/service/user_service.py index 05c0c5fd8c7..5ed35acc448 100644 --- a/openbb_platform/core/openbb_core/app/service/user_service.py +++ b/openbb_platform/core/openbb_core/app/service/user_service.py @@ -21,13 +21,11 @@ class UserService(metaclass=SingletonMeta): default_user_settings: Optional[UserSettings] = None, ): """Initialize user service.""" - self._default_user_settings = ( - default_user_settings or self.read_default_user_settings() - ) + self._default_user_settings = default_user_settings or self.read_from_file() @classmethod - def read_default_user_settings(cls, path: Optional[Path] = None) -> UserSettings: - """Read default user settings.""" + def read_from_file(cls, path: Optional[Path] = None) -> UserSettings: + """Read user settings from json into UserSettings.""" path = path or cls.USER_SETTINGS_PATH return ( @@ -37,16 +35,15 @@ class UserService(metaclass=SingletonMeta): ) @classmethod - def write_default_user_settings( + def write_to_file( cls, user_settings: UserSettings, path: Optional[Path] = None, ) -> None: - """Write default user settings.""" + """Write user settings to json.""" path = path or cls.USER_SETTINGS_PATH - user_settings_json = user_settings.model_dump_json( - include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, indent=4 + indent=4, include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, exclude_defaults=True ) path.write_text(user_settings_json, encoding="utf-8") diff --git a/openbb_platform/core/openbb_core/app/static/account.py b/openbb_platform/core/openbb_core/app/static/account.py index 8176c901052..d840b6b9fa3 100644 --- a/openbb_platform/core/openbb_core/app/static/account.py +++ b/openbb_platform/core/openbb_core/app/static/account.py @@ -156,9 +156,7 @@ class Account: # noqa: D205, D400 User settings: profile, credentials, preferences """ if not self._hub_service: - UserService.write_default_user_settings( - self._base_app._command_runner.user_settings - ) + UserService.write_to_file(self._base_app._command_runner.user_settings) else: self._hub_service.push(self._base_app._command_runner.user_settings) @@ -181,9 +179,7 @@ class Account: # noqa: D205, D400 User settings: profile, credentials, preferences """ if not self._hub_service: - self._base_app._command_runner.user_settings = ( - UserService.read_default_user_settings() - ) + self._base_app._command_runner.user_settings = UserService.read_from_file() else: incoming = self._hub_service.pull() self._base_app.user.profile = incoming.profile @@ -216,9 +212,7 @@ class Account: # noqa: D205, D400 if session_file.exists(): session_file.unlink() - self._base_app._command_runner.user_settings = ( - UserService.read_default_user_settings() - ) + self._base_app._command_runner.user_settings = UserService.read_from_file() if return_settings: return self._base_app._command_runner.user_settings diff --git a/openbb_platform/core/tests/app/service/test_hub_service.py b/openbb_platform/core/tests/app/service/test_hub_service.py index 903eef2059b..5499109e225 100644 --- a/openbb_platform/core/tests/app/service/test_hub_service.py +++ b/openbb_platform/core/tests/app/service/test_hub_service.py @@ -256,9 +256,9 @@ def test_hub2platform_v3_only(): """Test hub2platform.""" mock_user_settings = MagicMock(spec=HubUserSettings) mock_user_settings.features_keys = { - "API_KEY_FINANCIALMODELINGPREP": "abc", - "API_POLYGON_KEY": "def", - "API_FRED_KEY": "ghi", + "api_key_financialmodelingprep": "abc", + "api_polygon_key": "def", + "api_fred_key": "ghi", } mock_user_settings.features_settings = {} @@ -273,10 +273,10 @@ def test_hub2platform_v3v4(): """Test hub2platform.""" mock_user_settings = MagicMock(spec=HubUserSettings) mock_user_settings.features_keys = { - "API_KEY_FINANCIALMODELINGPREP": "abc", + "api_key_financialmodelingprep": "abc", "fmp_api_key": "other_key", - "API_POLYGON_KEY": "def", - "API_FRED_KEY": "ghi", + "api_polygon_key": "def", + "api_fred_key": "ghi", } mock_user_settings.features_settings = {} @@ -291,9 +291,9 @@ def test_platform2hub(): """Test platform2hub.""" mock_user_settings = MagicMock(spec=HubUserSettings) mock_user_settings.features_keys = { # Received from Hub - "API_KEY_FINANCIALMODELINGPREP": "abc", + "api_key_financialmodelingprep": "abc", "fmp_api_key": "other_key", - "API_FRED_KEY": "ghi", + "api_fred_key": "ghi", } mock_user_settings.features_settings = {} mock_hub_service = HubService() @@ -309,10 +309,10 @@ def test_platform2hub(): user_settings = mock_hub_service.platform2hub(mock_credentials, mock_defaults) assert isinstance(user_settings, HubUserSettings) - assert user_settings.features_keys["API_KEY_FINANCIALMODELINGPREP"] == "fmp" + assert user_settings.features_keys["api_key_financialmodelingprep"] == "fmp" assert user_settings.features_keys["fmp_api_key"] == "other_key" assert user_settings.features_keys["polygon_api_key"] == "polygon" - assert user_settings.features_keys["API_FRED_KEY"] == "fred" + assert user_settings.features_keys["api_fred_key"] == "fred" assert user_settings.features_keys["benzinga_api_key"] == "benzinga" assert "some_api_key" not in user_settings.features_keys assert "defaults" in user_settings.features_settings diff --git a/openbb_platform/core/tests/app/service/test_system_service.py b/openbb_platform/core/tests/app/service/test_system_service.py index 5ecac85bbbe..a563a1a3771 100644 --- a/openbb_platform/core/tests/app/service/test_system_service.py +++ b/openbb_platform/core/tests/app/service/test_system_service.py @@ -17,17 +17,19 @@ def test_system_service_init(system_service): assert system_service -def test_read_default_system_settings(system_service): +def test_read_from_file(system_service): """Test read default system settings.""" - system_settings = system_service._read_default_system_settings() + # pylint: disable=protected-access + system_settings = system_service._read_from_file() assert system_settings -def test_write_default_system_settings(system_service): +def test_write_to_file(system_service): """Test write default system settings.""" - system_settings = system_service._read_default_system_settings() - system_service.write_default_system_settings(system_settings=system_settings) + # pylint: disable=protected-access + system_settings = system_service._read_from_file() + system_service.write_to_file(system_settings=system_settings) assert system_service diff --git a/openbb_platform/core/tests/app/service/test_user_service.py b/openbb_platform/core/tests/app/service/test_user_service.py index cf64ff98de2..f00ac2a347f 100644 --- a/openbb_platform/core/tests/app/service/test_user_service.py +++ b/openbb_platform/core/tests/app/service/test_user_service.py @@ -10,15 +10,15 @@ from openbb_core.app.service.user_service import ( ) -def test_read_default_user_settings_file_exists(): +def test_read_from_file_file_exists(): """Test read default user settings.""" - result = UserService.read_default_user_settings(path=Path("some_path")) + result = UserService.read_from_file(path=Path("some_path")) assert result assert isinstance(result, UserSettings) -def test_write_default_user_settings(): +def test_write_to_file(): """Test write default user settings.""" # Create a temporary file for this test with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -31,7 +31,7 @@ def test_write_default_user_settings(): user_settings.defaults = {"language": "en"} # type: ignore[assignment] # Write the user settings to the temporary file - UserService.write_default_user_settings(user_settings, temp_path) + UserService.write_to_file(user_settings, temp_path) # Read the file and verify its contents with open(temp_path, encoding="utf-8") as file: diff --git a/openbb_platform/core/tests/app/static/test_container.py b/openbb_platform/core/tests/app/static/test_container.py index 711a9b3bfe7..a19b1331b09 100644 --- a/openbb_platform/core/tests/app/static/test_container.py +++ b/openbb_platform/core/tests/app/static/test_container.py @@ -89,7 +89,7 @@ def test_container__check_credentials(container): OpenBBError, escape( "Provider fallback failed." - "\n[Providers]\n * 'x' -> not found\n * 'y' -> not found\n * 'z' -> not found" + "\n[Providers]\n * 'x' -> not installed, please install openbb-x\n * 'y' -> not installed, please install openbb-y\n * 'z' -> not installed, please install openbb-z" ), ), ], |