diff options
47 files changed, 1607 insertions, 548 deletions
diff --git a/openbb_platform/core/openbb_core/app/model/obbject.py b/openbb_platform/core/openbb_core/app/model/obbject.py index 7251dc9969b..9f2a2da8746 100644 --- a/openbb_platform/core/openbb_core/app/model/obbject.py +++ b/openbb_platform/core/openbb_core/app/model/obbject.py @@ -113,13 +113,13 @@ class OBBject(Tagged, Generic[T]): return f"OBBject[{cls.results_type_repr(params)}]" def to_df( - self, index: Optional[str] = None, sort_by: Optional[str] = None + self, index: Optional[Union[str, None]] = "date", sort_by: Optional[str] = None ) -> pd.DataFrame: """Alias for `to_dataframe`.""" return self.to_dataframe(index=index, sort_by=sort_by) def to_dataframe( - self, index: Optional[str] = None, sort_by: Optional[str] = None + self, index: Optional[Union[str, None]] = "date", sort_by: Optional[str] = None ) -> pd.DataFrame: """Convert results field to pandas dataframe. @@ -173,7 +173,7 @@ class OBBject(Tagged, Generic[T]): for k, v in r.items(): # Dict[str, List[BaseModel]] if is_list_of_basemodel(v): - dict_of_df[k] = basemodel_to_df(v, index or "date") + dict_of_df[k] = basemodel_to_df(v, index) sort_columns = False # Dict[str, Any] else: @@ -184,14 +184,14 @@ class OBBject(Tagged, Generic[T]): # List[BaseModel] elif is_list_of_basemodel(res): dt: Union[List[Data], Data] = res # type: ignore - df = basemodel_to_df(dt, index or "date") + df = basemodel_to_df(dt, index) sort_columns = False # List[List | str | int | float] | Dict[str, Dict | List | BaseModel] else: try: df = pd.DataFrame(res) # Set index, if any - if index and index in df.columns: + if index is not None and index in df.columns: df.set_index(index, inplace=True) except ValueError: @@ -234,11 +234,11 @@ class OBBject(Tagged, Generic[T]): "Please install polars: `pip install polars pyarrow` to use this method." ) from exc - return from_pandas(self.to_dataframe().reset_index()) + return from_pandas(self.to_dataframe(index=None)) def to_numpy(self) -> ndarray: """Convert results field to numpy array.""" - return self.to_dataframe().reset_index().to_numpy() + return self.to_dataframe(index=None).to_numpy() def to_dict( self, @@ -259,7 +259,7 @@ class OBBject(Tagged, Generic[T]): Dict[str, List] Dictionary of lists. """ - df = self.to_dataframe() # type: ignore + df = self.to_dataframe(index=None) # type: ignore transpose = False if orient == "list": transpose = True diff --git a/openbb_platform/core/openbb_core/app/static/package_builder.py b/openbb_platform/core/openbb_core/app/static/package_builder.py index 52a16185467..3caa6d31974 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder.py @@ -79,7 +79,9 @@ class PackageBuilder: def auto_build(self) -> None: """Trigger build if there are differences between built and installed extensions.""" if Env().AUTO_BUILD: - add, remove = PackageBuilder._diff(self.directory / "package") + add, remove = PackageBuilder._diff( + self.directory / "assets" / "extension_map.json" + ) if add: a = ", ".join(add) print(f"Extensions to add: {a}") # noqa: T201 @@ -98,7 +100,7 @@ class PackageBuilder: ) -> None: """Build the extensions for the Platform.""" self.console.log("\nBuilding extensions package...\n") - self._clean_package(modules) + self._clean(modules) ext_map = self._get_extension_map() self._save_extension_map(ext_map) self._save_module_map() @@ -107,8 +109,9 @@ class PackageBuilder: if self.lint: self._run_linters() - def _clean_package(self, modules: Optional[Union[str, List[str]]] = None) -> None: - """Delete the package folder or modules before building.""" + def _clean(self, modules: Optional[Union[str, List[str]]] = None) -> None: + """Delete the assets and package folder or modules before building.""" + shutil.rmtree(self.directory / "assets", ignore_errors=True) if modules: for module in modules: module_path = self.directory / "package" / f"{module}.py" @@ -141,7 +144,7 @@ class PackageBuilder: """Save the map of extensions available at build time.""" code = dumps(obj=dict(sorted(ext_map.items())), indent=4) self.console.log("Writing extension map...") - self._write(code=code, name="extension_map", extension="json") + self._write(code=code, name="extension_map", extension="json", folder="assets") def _save_module_map(self): """Save the module map.""" @@ -152,7 +155,7 @@ class PackageBuilder: } code = dumps(obj=dict(sorted(module_map.items())), indent=4) self.console.log("\nWriting module map...") - self._write(code=code, name="module_map", extension="json") + self._write(code=code, name="module_map", extension="json", folder="assets") def _save_modules( self, @@ -197,9 +200,11 @@ class PackageBuilder: linters.ruff() linters.black() - def _write(self, code: str, name: str, extension="py") -> None: + def _write( + self, code: str, name: str, extension: str = "py", folder: str = "package" + ) -> None: """Write the module to the package.""" - package_folder = self.directory / "package" + package_folder = self.directory / folder package_path = package_folder / f"{name}.{extension}" package_folder.mkdir(exist_ok=True) @@ -209,25 +214,24 @@ class PackageBuilder: file.write(code.replace("typing.", "")) @staticmethod - def _read_extension_map(package: Path) -> dict: - """Get extension map from package folder.""" - ext_map_file = Path(package, "extension_map.json") + def _read(path: Path) -> dict: + """Get content from folder.""" try: - with open(ext_map_file) as fp: - ext_map = load(fp) + with open(Path(path)) as fp: + content = load(fp) except Exception: - ext_map = {} + content = {} - return ext_map + return content @staticmethod - def _diff(package: Path) -> Tuple[Set[str], Set[str]]: + def _diff(path: Path) -> Tuple[Set[str], Set[str]]: """Check differences between built and installed extensions. Parameters ---------- - package: Path - The path to the package + path: Path + The path to the folder where the extension map is stored. Returns ------- @@ -235,7 +239,7 @@ class PackageBuilder: First element: set of installed extensions that are not in the package. Second element: set of extensions in the package that are not installed. """ - ext_map = PackageBuilder._read_extension_map(package) + ext_map = PackageBuilder._read(path) add: Set[str] = set() remove: Set[str] = set() diff --git a/openbb_platform/core/openbb_core/app/utils.py b/openbb_platform/core/openbb_core/app/utils.py index b1a563131ac..4004e3060a9 100644 --- a/openbb_platform/core/openbb_core/app/utils.py +++ b/openbb_platform/core/openbb_core/app/utils.py @@ -2,6 +2,7 @@ import ast import json +from datetime import time from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -16,7 +17,7 @@ from openbb_core.provider.abstract.data import Data def basemodel_to_df( data: Union[List[Data], Data], - index: Optional[Union[str, Iterable]] = None, + index: Optional[Union[None, str, Iterable]] = None, ) -> pd.DataFrame: """Convert list of BaseModel to a Pandas DataFrame.""" if isinstance(data, list): @@ -32,12 +33,20 @@ def basemodel_to_df( df = df.set_index(col_names) df = df.drop(["is_multiindex", "multiindex_names"], axis=1) + # If the date column contains dates only, convert them to a date to avoid encoding time data. + if "date" in df.columns: + df["date"] = df["date"].apply(pd.to_datetime) + if all(t.time() == time(0, 0) for t in df["date"]): + df["date"] = df["date"].apply(lambda x: x.date()) + if index and index in df.columns: - df = df.set_index(index) - # TODO: This should probably check if the index can be converted to a datetime instead of just assuming - if df.index.name == "date": - df.index = pd.to_datetime(df.index) + if index == "date": + df.set_index("date", inplace=True) df.sort_index(axis=0, inplace=True) + else: + df = ( + df.set_index(index) if index is not None and index in df.columns else df + ) return df @@ -59,6 +68,12 @@ def df_to_basemodel( df["multiindex_names"] = str(df.index.names) df = df.reset_index() + # Converting to JSON will add T00:00:00.000 to all dates with no time element unless we format it as a string first. + if "date" in df.columns: + df["date"] = df["date"].apply(pd.to_datetime) + if all(t.time() == time(0, 0) for t in df["date"]): + df["date"] = df["date"].apply(lambda x: x.date().strftime("%Y-%m-%d")) + return [ Data(**d) for d in json.loads(df.to_json(orient="records", date_format="iso")) ] diff --git a/openbb_platform/core/openbb_core/provider/standard_models/crypto_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/crypto_historical.py index eeeda10a212..f3385d4932d 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/crypto_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/crypto_historical.py @@ -45,7 +45,9 @@ class CryptoHistoricalQueryParams(QueryParams): class CryptoHistoricalData(Data): """Crypto Historical Price Data.""" - date: datetime = Field(description=DATA_DESCRIPTIONS.get("date", "")) + date: Union[dateType, datetime] = Field( + description=DATA_DESCRIPTIONS.get("date", "") + ) open: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("open", "")) high: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("high", "")) low: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("low", "")) @@ -59,4 +61,6 @@ class CryptoHistoricalData(Data): @classmethod def date_validate(cls, v): # pylint: disable=E0213 """Return formatted datetime.""" - return parser.isoparse(str(v)) + if ":" in str(v): + return parser.isoparse(str(v)) + return parser.parse(str(v)).date() diff --git a/openbb_platform/core/openbb_core/provider/standard_models/currency_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/currency_historical.py index b7cedc20860..196ce3e1c37 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/currency_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/currency_historical.py @@ -46,7 +46,9 @@ class CurrencyHistoricalQueryParams(QueryParams): class CurrencyHistoricalData(Data): """Currency Historical Price Data.""" - date: datetime = Field(description=DATA_DESCRIPTIONS.get("date", "")) + date: Union[dateType, datetime] = Field( + description=DATA_DESCRIPTIONS.get("date", "") + ) open: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("open", "")) high: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("high", "")) low: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("low", "")) @@ -61,4 +63,6 @@ class CurrencyHistoricalData(Data): @field_validator("date", mode="before", check_fields=False) def date_validate(cls, v): # pylint: disable=E0213 """Return formatted datetime.""" - return parser.isoparse(str(v)) + if ":" in str(v): + return parser.isoparse(str(v)) + return parser.parse(str(v)).date() diff --git a/openbb_platform/core/openbb_core/provider/standard_models/equity_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/equity_historical.py index 2789e57ba35..e888928e683 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/equity_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/equity_historical.py @@ -61,7 +61,6 @@ class EquityHistoricalData(Data): @field_validator("date", mode="before", check_fields=False) def date_validate(cls, v): # pylint: disable=E0213 """Return formatted datetime.""" - v = parser.isoparse(str(v)) - if v.hour == 0 and v.minute == 0: - return v.date() - return v + if ":" in str(v): + return parser.isoparse(str(v)) + return parser.parse(str(v)).date() diff --git a/openbb_platform/core/openbb_core/provider/standard_models/etf_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/etf_historical.py index 6a2eb9d5dee..f9a4b5448ed 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/etf_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/etf_historical.py @@ -1,7 +1,10 @@ """ETF Historical Price Standard Model.""" -from datetime import date as dateType -from typing import Optional +from datetime import ( + date as dateType, + datetime, +) +from typing import Optional, Union from dateutil import parser from pydantic import Field, NonNegativeInt, PositiveFloat, field_validator @@ -37,7 +40,9 @@ class EtfHistoricalQueryParams(QueryParams): class EtfHistoricalData(Data): """ETF Historical Price Data.""" - date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", "")) + date: Union[dateType, datetime] = Field( + description=DATA_DESCRIPTIONS.get("date", "") + ) open: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("open", "")) high: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("high", "")) low: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("low", "")) @@ -49,4 +54,6 @@ class EtfHistoricalData(Data): @field_validator("date", mode="before", check_fields=False) def date_validate(cls, v): # pylint: disable=E0213 """Return formatted datetime.""" - return parser.isoparse(str(v)) + if ":" in str(v): + return parser.isoparse(str(v)) + return parser.parse(str(v)).date() diff --git a/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py index e6183bf9921..f9a381698cb 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py @@ -73,7 +73,6 @@ class IndexHistoricalData(Data): @classmethod def date_validate(cls, v): """Return formatted datetime.""" - v = parser.isoparse(str(v)) - if v.hour == 0 and v.minute == 0: - return v.date() - return v + if ":" in str(v): + return parser.isoparse(str(v)) + return parser.parse(str(v)).date() diff --git a/openbb_platform/core/tests/app/model/test_obbject.py b/openbb_platform/core/tests/app/model/test_obbject.py index 0e9692b8f70..ffe5b8368b2 100644 --- a/openbb_platform/core/tests/app/model/test_obbject.py +++ b/openbb_platform/core/tests/app/model/test_obbject.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock import pandas as pd import pytest from openbb_core.app.model.obbject import Chart, OBBject, OpenBBError +from openbb_core.app.utils import basemodel_to_df from openbb_core.provider.abstract.data import Data from pandas.testing import assert_frame_equal @@ -48,6 +49,13 @@ class MockMultiData(Data): value: float +class MockDataFrame(Data): + """Test helper.""" + + date: str + value: float + + @pytest.mark.parametrize( "results, expected_df", [ @@ -171,27 +179,27 @@ class MockMultiData(Data): "df1": pd.DataFrame( { "date": [ - pd.to_datetime("1956-01-01"), - pd.to_datetime("1956-02-01") |