From 6cfc47541ab51159d5e9761774e4c415ce4c36e6 Mon Sep 17 00:00:00 2001 From: Henrique Joaquim Date: Fri, 19 Apr 2024 16:00:02 +0100 Subject: [BugFix] Adding safe timestamp conversion everywhere (#6299) * safe fromtimestamp * sage timestamp convertion on equity profile * safe fromtimestamp * fix imports * Lint * Force UTC Currency Snapshots * Add that to market_snapshots too. * black * ignoring some typing * ignoring some typing * typing * typing * typing --------- Co-authored-by: hjoaquim Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com> Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com> --- .../core/openbb_core/provider/utils/helpers.py | 4 +++- .../openbb_benzinga/models/analyst_search.py | 20 +++++++++++++------- .../openbb_benzinga/models/price_target.py | 13 ++++++++----- .../fmp/openbb_fmp/models/currency_snapshots.py | 11 ++++++----- .../fmp/openbb_fmp/models/equity_quote.py | 22 ++++++++++++++-------- .../fmp/openbb_fmp/models/key_executives.py | 16 ++++++++++++---- .../fmp/openbb_fmp/models/market_indices.py | 2 ++ .../fmp/openbb_fmp/models/market_snapshots.py | 20 +++++++++++++++----- .../openbb_intrinio/models/market_snapshots.py | 8 ++++---- .../openbb_polygon/models/crypto_historical.py | 18 +++++++++++------- .../openbb_polygon/models/currency_historical.py | 18 +++++++++++------- .../openbb_polygon/models/equity_historical.py | 18 ++++++++++-------- .../openbb_polygon/models/index_historical.py | 14 +++++++------- .../openbb_polygon/models/market_indices.py | 8 ++++++-- .../openbb_tradier/models/equity_historical.py | 9 +++------ .../tradier/openbb_tradier/models/equity_quote.py | 9 ++++----- .../openbb_tradier/models/options_chains.py | 5 +++-- .../yfinance/openbb_yfinance/models/etf_info.py | 3 ++- 18 files changed, 134 insertions(+), 84 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/helpers.py b/openbb_platform/core/openbb_core/provider/utils/helpers.py index 36c5907ae7e..604323e9703 100644 --- a/openbb_platform/core/openbb_core/provider/utils/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/helpers.py @@ -316,7 +316,9 @@ def filter_by_dates( return list(filter(_filter, data)) -def safe_fromtimestamp(timestamp: float, tz: Optional[timezone] = None) -> datetime: +def safe_fromtimestamp( + timestamp: Union[float, int], tz: Optional[timezone] = None +) -> datetime: """datetime.fromtimestamp alternative which supports negative timestamps on Windows platform.""" if os.name == "nt" and timestamp < 0: return datetime(1970, 1, 1, tzinfo=tz) + timedelta(seconds=timestamp) diff --git a/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py b/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py index 1d9424309a0..f5cf40a2a5a 100644 --- a/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py +++ b/openbb_platform/providers/benzinga/openbb_benzinga/models/analyst_search.py @@ -2,7 +2,10 @@ # pylint: disable=unused-argument -from datetime import datetime +from datetime import ( + date as dateType, + timezone, +) from typing import Any, Dict, List, Optional from openbb_core.provider.abstract.fetcher import Fetcher @@ -11,9 +14,12 @@ from openbb_core.provider.standard_models.analyst_search import ( AnalystSearchQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request, get_querystring +from openbb_core.provider.utils.helpers import ( + amake_request, + get_querystring, + safe_fromtimestamp, +) from pydantic import Field, field_validator, model_validator -from pytz import UTC class BenzingaAnalystSearchQueryParams(AnalystSearchQueryParams): @@ -358,12 +364,12 @@ class BenzingaAnalystSearchData(AnalystSearchData): @field_validator("last_updated", mode="before", check_fields=False) @classmethod - def validate_date(cls, v): - """Validate date.""" + def validate_date(cls, v: float) -> Optional[dateType]: + """Validate last_updated.""" if v: - dt = datetime.fromtimestamp(v, UTC) + dt = safe_fromtimestamp(v, tz=timezone.utc) return dt.date() if dt.time() == dt.min.time() else dt - return v + return None @model_validator(mode="before") @classmethod diff --git a/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py b/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py index 561cb9862fa..d799e3410d8 100644 --- a/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py +++ b/openbb_platform/providers/benzinga/openbb_benzinga/models/price_target.py @@ -17,9 +17,12 @@ from openbb_core.provider.standard_models.price_target import ( ) from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_requests, get_querystring +from openbb_core.provider.utils.helpers import ( + amake_requests, + get_querystring, + safe_fromtimestamp, +) from pydantic import Field, field_validator, model_validator -from pytz import UTC COVERAGE_DICT = { "downgrades": "Downgrades", @@ -221,12 +224,12 @@ class BenzingaPriceTargetData(PriceTargetData): @field_validator("last_updated", mode="before", check_fields=False) @classmethod - def validate_date(cls, v): + def validate_date(cls, v: float) -> Optional[dateType]: """Convert the Unix timestamp to a datetime object.""" if v: - dt = datetime.fromtimestamp(v, UTC) + dt = safe_fromtimestamp(v, tz=timezone.utc) return dt.date() if dt.time() == dt.min.time() else dt - return v + return None @model_validator(mode="before") @classmethod diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py b/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py index 97ea456eb4b..6276e23ed73 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/currency_snapshots.py @@ -2,7 +2,10 @@ # pylint: disable=unused-argument -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Optional from openbb_core.provider.abstract.fetcher import Fetcher @@ -11,7 +14,7 @@ from openbb_core.provider.standard_models.currency_snapshots import ( CurrencySnapshotsQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from pandas import DataFrame, concat from pydantic import Field, field_validator @@ -85,7 +88,6 @@ class FMPCurrencySnapshotsFetcher( **kwargs: Any, ) -> List[Dict]: """Extract the data from the FMP endpoint.""" - api_key = credentials.get("fmp_api_key") if credentials else "" url = f"https://financialmodelingprep.com/api/v3/quotes/forex?apikey={api_key}" @@ -99,7 +101,6 @@ class FMPCurrencySnapshotsFetcher( **kwargs: Any, ) -> List[FMPCurrencySnapshotsData]: """Filter by the query parameters and validate the model.""" - if not data: raise EmptyDataError("No data was returned from the FMP endpoint.") @@ -143,7 +144,7 @@ class FMPCurrencySnapshotsFetcher( if len(temp) > 0: # Convert the Unix timestamp to a datetime. temp.timestamp = temp.timestamp.apply( - lambda x: datetime.fromtimestamp(x) + lambda x: safe_fromtimestamp(x, tz=timezone.utc) ) new_df = concat([new_df, temp]) if len(new_df) == 0: diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py b/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py index b5511730755..3830466770a 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/equity_quote.py @@ -1,8 +1,14 @@ """FMP Equity Quote Model.""" +# pylint: disable=unused-argument + import asyncio -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional +from datetime import ( + date as dateType, + datetime, + timezone, +) +from typing import Any, Dict, List, Optional, Union from warnings import warn from openbb_core.provider.abstract.data import ForceInt @@ -12,7 +18,7 @@ from openbb_core.provider.standard_models.equity_quote import ( EquityQuoteQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from openbb_fmp.utils.helpers import get_querystring, response_callback from pydantic import Field, field_validator @@ -63,22 +69,22 @@ class FMPEquityQuoteData(EquityQuoteData): @field_validator("last_timestamp", mode="before", check_fields=False) @classmethod - def validate_last_timestamp(cls, v): # pylint: disable=E0213 + def validate_last_timestamp(cls, v: Union[str, int]) -> Optional[dateType]: """Return the date as a datetime object.""" if v: v = int(v) if isinstance(v, str) else v - return datetime.fromtimestamp(int(v), tz=timezone.utc) + return safe_fromtimestamp(v, tz=timezone.utc) return None @field_validator("earnings_announcement", mode="before", check_fields=False) @classmethod - def timestamp_validate(cls, v): # pylint: disable=E0213 + def timestamp_validate(cls, v: str) -> Optional[dateType]: """Return the datetime string as a datetime object.""" if v: dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z") dt = dt.replace(microsecond=0) timestamp = dt.timestamp() - return datetime.fromtimestamp(timestamp, tz=timezone.utc) + return safe_fromtimestamp(timestamp, tz=timezone.utc) return None @field_validator("change_percent", mode="after", check_fields=False) @@ -115,7 +121,7 @@ class FMPEquityQuoteFetcher( symbols = query.symbol.split(",") - results = [] + results: list = [] async def get_one(symbol): """Get data for one symbol.""" diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py b/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py index 2329c7a7603..ab0f11d3283 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/key_executives.py @@ -1,13 +1,18 @@ """FMP Key Executives Model.""" -from datetime import datetime -from typing import Any, Dict, List, Optional +# pylint: disable=unused-argument + +from datetime import ( + date as dateType, +) +from typing import Any, Dict, List, Optional, Union from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.standard_models.key_executives import ( KeyExecutivesData, KeyExecutivesQueryParams, ) +from openbb_core.provider.utils.helpers import safe_fromtimestamp from openbb_fmp.utils.helpers import get_data_many from pydantic import field_validator @@ -24,9 +29,12 @@ class FMPKeyExecutivesData(KeyExecutivesData): @field_validator("titleSince", mode="before", check_fields=False) @classmethod - def time_validate(cls, v): # pylint: disable=E0213 + def time_validate(cls, v: Union[float, int]) -> Optional[dateType]: """Return the date as a datetime object.""" - return datetime.fromtimestamp(v / 1000) + if v: + v = v / 1000 + return safe_fromtimestamp(v) + return v # type: ignore class FMPKeyExecutivesFetcher( diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py b/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py index 7d860c4955d..15e4bb6719c 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/market_indices.py @@ -1,5 +1,7 @@ """FMP Market Indices Model.""" +# pylint: disable=unused-argument + from datetime import datetime from typing import Any, Dict, List, Literal, Optional diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py b/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py index 89bdff1f42e..a1a00e372e4 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/market_snapshots.py @@ -1,8 +1,11 @@ """FMP Market Snapshots Model.""" +# pylint: disable=unused-argument + from datetime import ( date as dateType, datetime, + timezone, ) from typing import Any, Dict, List, Optional, Union @@ -13,6 +16,7 @@ from openbb_core.provider.standard_models.market_snapshots import ( MarketSnapshotsData, MarketSnapshotsQueryParams, ) +from openbb_core.provider.utils.helpers import safe_fromtimestamp from openbb_fmp.utils.definitions import EXCHANGES from openbb_fmp.utils.helpers import get_data from pydantic import Field, field_validator @@ -87,14 +91,20 @@ class FMPMarketSnapshotsData(MarketSnapshotsData): @field_validator("last_price_timestamp", mode="before", check_fields=False) @classmethod - def validate_timestamp(cls, v): + def validate_timestamp(cls, v: Union[str, int, float]) -> Optional[dateType]: """Validate the timestamp.""" + if isinstance(v, str): + try: + v = float(v) + except ValueError: + return None + if isinstance(v, (int, float)) and v != 0: try: - v = datetime.fromtimestamp(v) - if v.hour == 0 and v.minute == 0 and v.second == 0: - v = v.date() - return v + v = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore + if v.hour == 0 and v.minute == 0 and v.second == 0: # type: ignore + v = v.date() # type: ignore + return v # type: ignore except ValueError: return None return None diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py index 2cb54996683..3e8f24879f5 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/market_snapshots.py @@ -7,6 +7,7 @@ import gzip from datetime import ( date as dateType, datetime, + timezone as datetime_timezone, ) from io import BytesIO from typing import Any, Dict, List, Optional, Union @@ -16,7 +17,7 @@ from openbb_core.provider.standard_models.market_snapshots import ( MarketSnapshotsData, MarketSnapshotsQueryParams, ) -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from pandas import DataFrame, notna, read_csv, to_datetime from pydantic import Field from pytz import timezone @@ -105,7 +106,7 @@ class IntrinioMarketSnapshotsFetcher( dt = transformed_params["date"] dt = dt.astimezone(tz=timezone("America/New_York")) if isinstance(transformed_params["date"], dateType): - dt = transformed_params["date"] + dt = transformed_params["date"] # type: ignore if isinstance(dt, dateType): dt = datetime( dt.year, @@ -143,7 +144,6 @@ class IntrinioMarketSnapshotsFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Intrinio endpoint.""" - api_key = credentials.get("intrinio_api_key") if credentials else "" # This gets the URL to the actual file. @@ -216,7 +216,7 @@ class IntrinioMarketSnapshotsFetcher( to_datetime( df[col].apply( lambda x: ( - datetime.fromtimestamp(x, tz=timezone("UTC")) + safe_fromtimestamp(x, tz=datetime_timezone.utc) if notna(x) else x ) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py index 04a566a6250..dc9bb95fe2b 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/crypto_historical.py @@ -3,7 +3,10 @@ # pylint: disable=unused-argument,protected-access,line-too-long import warnings -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +21,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -25,7 +29,6 @@ from pydantic import ( PrivateAttr, model_validator, ) -from pytz import timezone _warn = warnings.warn @@ -146,17 +149,18 @@ class PolygonCryptoHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore + results: list = data.get("results", []) # type: ignore while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore + next_url = data.get("next_url", None) # type: ignore for r in results: - r["t"] = datetime.fromtimestamp(r["t"] / 1000, tz=timezone("UTC")) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py index 0da2b73c2f2..9a4edcb98b4 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/currency_historical.py @@ -3,7 +3,10 @@ # pylint: disable=unused-argument,protected-access,line-too-long import warnings -from datetime import datetime +from datetime import ( + datetime, + timezone, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +21,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -25,7 +29,6 @@ from pydantic import ( PrivateAttr, model_validator, ) -from pytz import timezone _warn = warnings.warn @@ -143,17 +146,18 @@ class PolygonCurrencyHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore[union-attr] + results: list = data.get("results", []) # type: ignore[union-attr] while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore[union-attr] + next_url = data.get("next_url", None) # type: ignore[union-attr] for r in results: - r["t"] = datetime.fromtimestamp(r["t"] / 1000, tz=timezone("UTC")) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone.utc) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py index 85f4625a875..de818d61bfe 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/equity_historical.py @@ -3,7 +3,9 @@ # pylint: disable=unused-argument,protected-access import warnings -from datetime import datetime +from datetime import ( + datetime, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -18,6 +20,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pandas import to_datetime from pydantic import ( @@ -152,19 +155,18 @@ class PolygonEquityHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore + results: list = data.get("results", []) # type: ignore while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore + next_url = data.get("next_url", None) # type: ignore for r in results: - r["t"] = datetime.fromtimestamp( - r["t"] / 1000, tz=timezone("America/New_York") - ) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone("America/New_York")) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py b/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py index 01320ddfc75..74fa010e841 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/index_historical.py @@ -18,6 +18,7 @@ from openbb_core.provider.utils.helpers import ( ClientResponse, ClientSession, amake_requests, + safe_fromtimestamp, ) from pydantic import ( Field, @@ -143,19 +144,18 @@ class PolygonIndexHistoricalFetcher( data = await response.json() symbol = response.url.parts[4] - next_url = data.get("next_url", None) - results: list = data.get("results", []) + next_url = data.get("next_url", None) # type: ignore[union-attr] + results: list = data.get("results", []) # type: ignore[union-attr] while next_url: url = f"{next_url}&apiKey={api_key}" data = await session.get_json(url) - results.extend(data.get("results", [])) - next_url = data.get("next_url", None) + results.extend(data.get("results", [])) # type: ignore[union-attr] + next_url = data.get("next_url", None) # type: ignore[union-attr] for r in results: - r["t"] = datetime.fromtimestamp( - r["t"] / 1000, tz=timezone("America/New_York") - ) + v = r["t"] / 1000 # milliseconds to seconds + r["t"] = safe_fromtimestamp(v, tz=timezone("America/New_York")) # type: ignore[arg-type] if query._timespan not in ["second", "minute", "hour"]: r["t"] = r["t"].date().strftime("%Y-%m-%d") else: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/market_indices.py b/openbb_platform/providers/polygon/openbb_polygon/models/market_indices.py index b473a42cffb..35670008e50 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/market_indices.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/market_indices.py @@ -1,6 +1,8 @@ """Polygon Market Indices Model.""" -from datetime import datetime +from datetime import ( + datetime, +) from typing import Any, Dict, List, Literal, Optional from dateutil.relativedelta import relativedelta @@ -10,6 +12,7 @@ from openbb_core.provider.standard_models.market_indices import ( MarketIndicesQueryParams, ) from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS +from openbb_core.provider.utils.helpers import safe_fromtimestamp from openbb_polygon.utils.helpers import get_data_many from pydantic import Field, PositiveInt @@ -94,7 +97,8 @@ class PolygonMarketIndicesFetcher( data = await get_data_many(request_url, "results", **kwargs) for d in data: - d["t"] = datetime.fromtimestamp(d["t"] / 1000) + v = d["t"] / 1000 # milliseconds to seconds + d["t"] = safe_fromtimestamp(v) if query.timespan not in ["minute", "hour"]: d["t"] = d["t"].date() diff --git a/openbb_platform/providers/tradier/openbb_tradier/models/equity_historical.py b/openbb_platform/providers/tradier/openbb_tradier/models/equity_historical.py index b03249fc0d7..eed0d09d93a 100644 --- a/openbb_platform/providers/tradier/openbb_tradier/models/equity_historical.py +++ b/openbb_platform/providers/tradier/openbb_tradier/models/equity_historical.py @@ -1,4 +1,4 @@ -"""Tradier Equity Historical Model""" +"""Tradier Equity Historical Model.""" # pylint: disable = unused-argument @@ -16,7 +16,7 @@ from openbb_core.provider.utils.descriptions import ( QUERY_DESCRIPTIONS, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from openbb_tradier.utils.constants import INTERVALS_DICT from pandas import to_datetime from pydantic import Field @@ -56,7 +56,6 @@ class TradierEquityHistoricalFetcher( @staticmethod def transform_query(params: Dict[str, Any]) -> TradierEquityHistoricalQueryParams: """Transform the query.""" - if params.get("interval") in ["1d", "1W", "1M"]: if params.get("start_date") is None: params["start_date"] = (datetime.now() - timedelta(days=365)).date() @@ -85,7 +84,6 @@ class TradierEquityHistoricalFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Tradier endpoint.""" - api_key = credentials.get("tradier_api_key") if credentials else "" sandbox = True @@ -120,7 +118,6 @@ class TradierEquityHistoricalFetcher( async def get_one(symbol): """Get data for one symbol.""" - result = [] url = ( @@ -150,7 +147,7 @@ class TradierEquityHistoricalFetcher( r["symbol"] = symbol _ = r.pop("time") r["timestamp"] = ( - datetime.fromtimestamp(r.get("timestamp")) + safe_fromtimestamp(r.get("timestamp")) .replace(microsecond=0) .astimezone(timezone("America/New_York")) ) diff --git a/openbb_platform/providers/tradier/openbb_tradier/models/equity_quote.py b/openbb_platform/providers/tradier/openbb_tradier/models/equity_quote.py index 667879bcdf9..d3a2abef9fa 100644 --- a/openbb_platform/providers/tradier/openbb_tradier/models/equity_quote.py +++ b/openbb_platform/providers/tradier/openbb_tradier/models/equity_quote.py @@ -1,4 +1,4 @@ -"""Tradier Equity Quote Model""" +"""Tradier Equity Quote Model.""" # pylint: disable = unused-argument @@ -15,7 +15,7 @@ from openbb_core.provider.standard_models.equity_quote import ( EquityQuoteQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from openbb_tradier.utils.constants import OPTIONS_EXCHANGES, STOCK_EXCHANGES from pydantic import Field, field_validator, model_validator from pytz import timezone @@ -157,7 +157,8 @@ class TradierEquityQuoteData(EquityQuoteData): def validate_dates(cls, v): """Validate the dates.""" if v != 0 and v is not None and isinstance(v, int): - v = datetime.fromtimestamp(int(v) / 1000) + v = int(v) / 1000 # milliseconds to seconds + v = safe_fromtimestamp(v) v = v.replace(microsecond=0) v = v.astimezone(timezone("America/New_York")) return v @@ -202,7 +203,6 @@ class TradierEquityQuoteFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Tradier endpoint.""" - api_key = credentials.get("tradier_api_key") if credentials else "" sandbox = True @@ -245,7 +245,6 @@ class TradierEquityQuoteFetcher( **kwargs: Any, ) -> List[TradierEquityQuoteData]: """Transform and validate the data.""" - results: List[TradierEquityQuoteData] = [] for d in data: diff --git a/openbb_platform/providers/tradier/openbb_tradier/models/options_chains.py b/openbb_platform/providers/tradier/openbb_tradier/models/options_chains.py index edbfbe7557c..16cdbab51f0 100644 --- a/openbb_platform/providers/tradier/openbb_tradier/models/options_chains.py +++ b/openbb_platform/providers/tradier/openbb_tradier/models/options_chains.py @@ -13,7 +13,7 @@ from openbb_core.provider.standard_models.options_chains import ( OptionsChainsQueryParams, ) from openbb_core.provider.utils.errors import EmptyDataError -from openbb_core.provider.utils.helpers import amake_request +from openbb_core.provider.utils.helpers import amake_request, safe_fromtimestamp from openbb_tradier.utils.constants import OPTIONS_EXCHANGES, STOCK_EXCHANGES from pydantic import Field, field_validator, model_validator from pytz import timezone @@ -126,7 +126,8 @@ class TradierOptionsChainsData(OptionsChainsData): def validate_dates(cls, v): """Validate the dates.""" if v != 0 and v is not None and isinstance(v, int): - v = datetime.fromtimestamp(int(v) / 1000) + v = int(v) / 1000 # milliseconds to seconds + v = safe_fromtimestamp(v) v = v.replace(microsecond=0) v = v.astimezone(timezone("America/New_York")) return v diff --git a/openbb_platform/providers/yfinance/openbb_yfinance/models/etf_info.py b/openbb_platform/providers/yfinance/openbb_yfinance/models/etf_info.py index c2ce1a28355..915fb44bf4f 100644 --- a/openbb_platform/providers/yfinance/openbb_yfinance/models/etf_info.py +++ b/openbb_platform/providers/yfinance/openbb_yfinance/models/etf_info.py @@ -11,6 +11,7 @@ from openbb_core.provider.standard_models.etf_info import ( EtfInfoData, EtfInfoQueryParams, ) +from openbb_core.provider.utils.helpers import safe_fromtimestamp from pydantic import Field, field_validator from yfinance import Ticker @@ -275,7 +276,7 @@ class YFinanceEtfInfoFetcher( "fundInceptionDate" not in result and _first_trade is not None ): - result["fundInceptionDate"] = datetime.fromtimestamp( + result["fundInceptionDate"] = safe_fromtimestamp( _first_trade ) except Exception as e: -- cgit v1.2.3