diff options
author | Henrique Joaquim <henriquecjoaquim@gmail.com> | 2024-04-19 16:00:02 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-04-19 15:00:02 +0000 |
commit | 6cfc47541ab51159d5e9761774e4c415ce4c36e6 (patch) | |
tree | 49892c370a4abc2351c334a6a9c83884e23ce202 | |
parent | 8ab4be9ec526372681b040391953bd830cf2bb75 (diff) |
[BugFix] Adding safe timestamp conversion everywhere (#6299)
* safe fromtimestamp
* sage timestamp convertion on equity profile
* safe fromtimestamp
* fix imports
* Lint
* Force UTC Currency Snapshots
* Add that to market_snapshots too.
* black
* ignoring some typing
* ignoring some typing
* typing
* typing
* typing
---------
Co-authored-by: hjoaquim <h.joaquim@campus.fct.unl.pt>
Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
18 files changed, 134 insertions, 84 deletions
diff --git a/openbb_platform/core/openbb_core/provider/utils/helpers.py b/openbb_platform/core/openbb_core/provider/utils/helpers.py index 36c5907ae7e..604323e9703 100644 --- a/openbb_platform/core/openbb_core/provider/utils/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/helpers.py @@ -316,7 +316,9 @@ def filter_by_dates( return list(filter(_filter, data)) -def safe_fromtimestamp(timestamp: float, tz: Optional[timezone] = None) -> datetime: +def safe_fromtimestamp( + timestamp: Union[float, int], tz: Optional[timezone] = None +) -> datetime: """datetime.fromtimestamp alternative which supports negative timestamps on Windows platform.""" if os.name == "nt" and timestamp < 0: return datetime(1970, 1, 1, tzinfo=tz) + timedelta(seconds=timestamp) diff --git a/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py b/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py index 1d9424309a0..f5cf40a2a5a 100644 --- a/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py +++ b/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py @@ -2,7 +2,10 @@ # pylint: disable=unused-argument -from datetime import datetime +from datetime import ( + date as dateType, + timezone, +) from typing import Any, Dict, List, Optional from openbb_core.provider.abstract.fetcher import Fetcher @@ -11,9 +14,12 @@ from openbb_core.provider.standard_models.analyst_search import ( AnalystSearchQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request, get_querystring +from openbb_core.provider.utils.helpers import ( + amake_request, + get_querystring, + safe_fromtimestamp, +) from pydantic import Field, field_validator, model_validator -from pytz import UTC class BenzingaAnalystSearchQueryParams(AnalystSearchQueryParams): @@ -358,12 +364,12 @@ class BenzingaAnalystSearchData(AnalystSearchData): @field_validator("last_updated", mode="before", check_fields=False) @classmethod - def validate_date(cls, v): - """Validate date.""" + def validate_date(cls, v: float) -> Optional[dateType]: + """Validate last_updated.""" if v: - dt = datetime.fromtimestamp(v, UTC) + dt = safe_fromtimestamp(v, tz=timezone.utc) return dt.date() if dt.time() == dt.min.time() else dt - return v + return None @model_validator(mode="before") @classmethod diff --git a/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py b/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py index 561cb9862fa..d799e3410d8 100644 --- a/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py +++ b/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py @@ -17,9 +17,12 @@ from openbb_core.provider.standard_models.price_target import ( ) from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_requests, get_querystring +from openbb_core.provider.utils.helpers import ( + amake_requests, + get_querystring, + safe_fromtimestamp, +) from pydantic import Field, field_validator, model_validator -from pytz import UTC COVERAGE_DICT = { "downgrades": "Downgrades", @@ -221,12 +224,12 @@ class BenzingaPriceTargetData(PriceTargetData): @field_validator("last_updated", mode="before", check_fields=False) @classmethod - def validate_date(cls, v): + def validate_date(cls, v: float) -> Optional[dateType]: """Convert the Unix timestamp to a datetime object.""" if v: - dt = datetime.fromtimestamp(v, UTC) + dt = safe_fromtimestamp(v, tz=timezone.utc) return dt.date() if dt.time() == dt.min.time() else dt - return v + return None @model_validator(mode="before") @classmethod diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py b/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py index 97ea456eb4b..6276e23ed73 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py @@ -2,7 +2,10 @@ # pylint: disable=unused-argument -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Optional from openbb_core.provider.abstract.fetcher import Fetcher @@ -11,7 +14,7 @@ from openbb_core.provider.standard_models.currency_snapshots import ( CurrencySnapshotsQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from pandas import DataFrame, concat from pydantic import Field, field_validator @@ -85,7 +88,6 @@ class FMPCurrencySnapshotsFetcher( **kwargs: Any, ) -> List[Dict]: """Extract the data from the FMP endpoint.""" - api_key = credentials.get("fmp_api_key") if credentials else "" url = f"https://financialmodelingprep.com/api/v3/quotes/forex?apikey={api_key}" @@ -99,7 +101,6 @@ class FMPCurrencySnapshotsFetcher( **kwargs: Any, ) -> List[FMPCurrencySnapshotsData]: """Filter by the query parameters and validate the model.""" - if not data: raise EmptyDataError("No data was returned from the FMP endpoint.") @@ -143,7 +144,7 @@ class FMPCurrencySnapshotsFetcher( if len(temp) > 0: # Convert the Unix timestamp to a datetime. temp.timestamp = temp.timestamp.apply( - lambda x: datetime.fromtimestamp(x) + lambda x: safe_fromtimestamp(x, tz=timezone.utc) ) new_df = concat([new_df, temp]) if len(new_df) == 0: diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py b/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py index b5511730755..3830466770a 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py @@ -1,8 +1,14 @@ """FMP Equity Quote Model.""" +# pylint: disable=unused-argument + import asyncio -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional +from datetime import ( + date as dateType, + datetime, + timezone, +) +from typing import Any, Dict, List, Optional, Union from warnings import warn from openbb_core.provider.abstract.data import ForceInt @@ -12,7 +18,7 @@ from openbb_core.provider.standard_models.equity_quote import ( EquityQuoteQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from openbb_fmp.utils.helpers import get_querystring, response_callback from pydantic import Field, field_validator @@ -63,22 +69,22 @@ class FMPEquityQuoteData(EquityQuoteData): @field_validator("last_timestamp", mode="before", check_fields=False) @classmethod - def validate_last_timestamp(cls, v): # pylint: disable=E0213 + def validate_last_timestamp(cls, v: Union[str, int]) -> Optional[dateType]: """Return the date as a datetime object.""" if v: v = int(v) if isinstance(v, str) else v - return datetime.fromtimestamp(int(v), tz=timezone.utc) + return safe_fromtimestamp(v, tz=timezone.utc) return None @field_validator("earnings_announcement", mode="before", check_fields=False) @classmethod - def timestamp_validate(cls, v): # pylint: disable=E0213 + def timestamp_validate(cls, v: str) -> Optional[dateType]: """Return the datetime string as a datetime object.""" if v: dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z") dt = dt.replace(microsecond=0) timestamp = dt.timestamp() - return datetime.fromtimestamp(timestamp, tz=timezone.utc) + return safe_fromtimestamp(timestamp, tz=timezone.utc) return None @field_validator("change_percent", mode="after", check_fields=False) @@ -115,7 +121,7 @@ class FMPEquityQuoteFetcher( symbols = query.symbol.split(",") - results = [] + results: list = [] async def get_one(symbol): """Get data for one symbol.""" diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py b/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py index 2329c7a7603..ab0f11d3283 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py @@ -1,13 +1,18 @@ """FMP Key Executives Model.""" -from datetime import datetime -from typing import Any, Dict, List, Optional +# pylint: disable=unused-argument + +from datetime import ( + date as dateType, +) +from typing import Any, Dict, List, Optional, Union from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.standard_models.key_executives import ( KeyExecutivesData, KeyExecutivesQueryParams, ) +from openbb_core.provider.utils.helpers import safe_fromtimestamp from openbb_fmp.utils.helpers import get_data_many from pydantic import field_validator @@ -24,9 +29,12 @@ class FMPKeyExecutivesData(KeyExecutivesData): @field_validator("titleSince", mode="before", check_fields=False) @classmethod - def time_validate(cls, v): # pylint: disable=E0213 + def time_validate(cls, v: Union[float, int]) -> Optional[dateType]: """Return the date as a datetime object.""" - return datetime.fromtimestamp(v / 1000) + if v: + v = v / 1000 + return safe_fromtimestamp(v) + return v # type: ignore class FMPKeyExecutivesFetcher( diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py b/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py index 7d860c4955d..15e4bb6719c 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py @@ -1,5 +1,7 @@ """FMP Market Indices Model.""" +# pylint: disable=unused-argument + from datetime import datetime from typing import Any, Dict, List, Literal, Optional diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py b/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py index 89bdff1f42e..a1a00e372e4 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py @@ -1,8 +1,11 @@ """FMP Market Snapshots Model.""" +# pylint: disable=unused-argument + from datetime import ( date as dateType, datetime, + timezone, ) from typing import Any, Dict, List, Optional, Union @@ -13,6 +16,7 @@ from openbb_core.provider.standard_models.market_snapshots import ( MarketSnapshotsData, MarketSnapshotsQueryParams, ) +from openbb_core.provider.utils.helpers import safe_fromtimestamp from openbb_fmp.utils.definitions import EXCHANGES from openbb_fmp.utils.helpers import get_data from pydantic import Field, field_validator @@ -87,14 +91,20 @@ class FMPMarketSnapshotsData(MarketSnapshotsData): @field_validator("last_price_timestamp", mode="before", check_fields=False) @classmethod - def validate_timestamp(cls, v): + def validate_timestamp(cls, v: Union[str, int, float]) -> Optional[dateType]: """Validate the timestamp.""" + if isinstance(v, str): + try: + v = float(v) + except ValueError: + return None + if isinstance(v, (int, float)) and v != 0: try: - v = datetime.fromtimestamp(v) - if v.hour == 0 and v.minute == 0 and v.second == 0: - v = v.date() - return v + v = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore + if v.hour == 0 and v.minute == 0 and v.second == 0: # type: ignore + v = v.date() # type: ignore + return v # type: ignore except ValueError: return None return None diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py index 2cb54996683..3e8f24879f5 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py @@ -7,6 +7,7 @@ import gzip from datetime import ( date as dateType, datetime, + timezone as datetime_timezone, ) from io import BytesIO from typing import Any, Dict, List, Optional, Union @@ -16,7 +17,7 @@ from openbb_core.provider.standard_models.market_snapshots import ( MarketSnapshotsData, MarketSnapshotsQueryParams, ) -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from pandas import DataFrame, notna, read_csv, to_datetime from pydantic import Field from pytz import timezone @@ -105,7 +106,7 @@ class IntrinioMarketSnapshotsFetcher( dt = transformed_params["date"] dt = dt.astimezone(tz=timezone("America/New_York")) if isinstance(transformed_params["date"], dateType): - dt = transformed_params["date"] + dt = transformed_params["date"] # type: ignore if isinstance(dt, dateType): dt = datetime( dt.year, @@ -143,7 +144,6 @@ class IntrinioMarketSnapshotsFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Intrinio endpoint.""" - api_key = credentials.get("intrinio_api_key") if credentials else "" # This gets the URL to the actual file. @@ -216,7 +216,7 @@ class IntrinioMarketSnapshotsFetcher( to_datetime( df[col].apply( lambda x: ( - datetime.fromtimestamp(x, tz=timezone("UTC")) + safe_fromtimestamp(x, tz=datetime_timezone.utc) if notna(x) else x ) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py index 04a566a6250..dc9bb95fe2b 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py @@ -3,7 +3,10 @@ # pylint: disable=unused-argument,protected-access,line-too-long import warnings -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +21,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -25,7 +29,6 @@ from pydantic import ( PrivateAttr, model_validator, ) -from pytz import timezone _warn = warnings.warn @@ -146,17 +149,18 @@ class PolygonCryptoHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore + results: list = data.get("results", []) # type: ignore while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore + next_url = data.get("next_url", None) # type: ignore for r in results: - r["t"] = datetime.fromtimestamp(r["t"] / 1000, tz=timezone("UTC")) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py index 0da2b73c2f2..9a4edcb98b4 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py @@ -3,7 +3,10 @@ # pylint: disable=unused-argument,protected-access,line-too-long import warnings -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +21,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -25,7 +29,6 @@ from pydantic import ( PrivateAttr, model_validator, ) -from pytz import timezone _warn = warnings.warn @@ -143,17 +146,18 @@ class PolygonCurrencyHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore[union-attr] + results: list = data.get("results", []) # type: ignore[union-attr] while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore[union-attr] + next_url = data.get("next_url", None) # type: ignore[union-attr] for r in results: - r["t"] = datetime.fromtimestamp(r["t"] / 1000, tz=timezone("UTC")) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py index 85f4625a875..de818d61bfe 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py @@ -3,7 +3,9 @@ # pylint: disable=unused-argument,protected-access import warnings -from datetime import datetime +from datetime import ( + datetime, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +20,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pandas import to_datetime from pydantic import ( @@ -152,19 +155,18 @@ class PolygonEquityHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore + results: list = data.get("results", []) # type: ignore while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore + next_url = data.get("next_url", None) # type: ignore for r in results: - r["t"] = datetime.fromtimestamp( - r["t"] / 1000, tz=timezone("America/New_York") - ) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone("America/New_York")) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py index 01320ddfc75..74fa010e841 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py @@ -18,6 +18,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -143,19 +144,18 @@ class PolygonIndexHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - result |