diff options
author | Danglewood <85772166+deeleeramone@users.noreply.github.com> | 2024-05-14 03:34:48 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-14 10:34:48 +0000 |
commit | 17a7e7d263eeab112bc478938afdaacfe4cb5e40 (patch) | |
tree | 8b76e6d58b86d8107c57ff323a53690e62222fbd | |
parent | 2ac1af3f265d7b6c863d9b2bb86ba643819cc969 (diff) |
[BugFix] Econ Calendar (#6392)
* fix econ calendar
* black
* more black
* pylint
* add 1 to n_urls
* set default dates in transform query
* missing decorator
* add None to literal
* literals
---------
Co-authored-by: Henrique Joaquim <henriquecjoaquim@gmail.com>
Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
10 files changed, 2971 insertions, 337 deletions
diff --git a/openbb_platform/core/openbb_core/provider/standard_models/economic_calendar.py b/openbb_platform/core/openbb_core/provider/standard_models/economic_calendar.py index 46aa718024b..67a268ce2b2 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/economic_calendar.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/economic_calendar.py @@ -4,7 +4,7 @@ from datetime import ( date as dateType, datetime, ) -from typing import Literal, Optional, Union +from typing import Optional, Union from pydantic import Field @@ -36,30 +36,26 @@ class EconomicCalendarData(Data): default=None, description=DATA_DESCRIPTIONS.get("date", "") ) country: Optional[str] = Field(default=None, description="Country of event.") + category: Optional[str] = Field(default=None, description="Category of event.") event: Optional[str] = Field(default=None, description="Event name.") - reference: Optional[str] = Field( - default=None, - description="Abbreviated period for which released data refers to.", + importance: Optional[str] = Field( + default=None, description="The importance level for the event." ) source: Optional[str] = Field(default=None, description="Source of the data.") - sourceurl: Optional[str] = Field(default=None, description="Source URL.") - actual: Optional[Union[str, float]] = Field( - default=None, description="Latest released value." + currency: Optional[str] = Field(default=None, description="Currency of the data.") + unit: Optional[str] = Field(default=None, description="Unit of the data.") + consensus: Optional[Union[str, float]] = Field( + default=None, + description="Average forecast among a representative group of economists.", ) previous: Optional[Union[str, float]] = Field( default=None, description="Value for the previous period after the revision (if revision is applicable).", ) - consensus: Optional[Union[str, float]] = Field( + revised: Optional[Union[str, float]] = Field( default=None, - description="Average forecast among a representative group of economists.", + description="Revised previous value, if applicable.", ) - forecast: Optional[Union[str, float]] = Field( - default=None, description="Trading Economics projections" - ) - url: Optional[str] = Field(default=None, description="Trading Economics URL") - importance: Optional[Union[Literal[0, 1, 2, 3], str]] = Field( - default=None, description="Importance of the event. 1-Low, 2-Medium, 3-High" + actual: Optional[Union[str, float]] = Field( + default=None, description="Latest released value." ) - currency: Optional[str] = Field(default=None, description="Currency of the data.") - unit: Optional[str] = Field(default=None, description="Unit of the data.") diff --git a/openbb_platform/extensions/economy/integration/test_economy_api.py b/openbb_platform/extensions/economy/integration/test_economy_api.py index 54ea5f6148f..f8de0ee5ca1 100644 --- a/openbb_platform/extensions/economy/integration/test_economy_api.py +++ b/openbb_platform/extensions/economy/integration/test_economy_api.py @@ -39,13 +39,16 @@ def headers(): "start_date": "2023-01-01", "end_date": "2023-06-06", "country": "mexico,sweden", - "importance": "Low", + "importance": "low", "group": "gdp", + "calendar_id": None, } ), ( { "provider": "fmp", + "start_date": "2023-10-24", + "end_date": "2023-11-03", } ), ], diff --git a/openbb_platform/extensions/economy/integration/test_economy_python.py b/openbb_platform/extensions/economy/integration/test_economy_python.py index 9a9eb1e35bd..dec33dacb6e 100644 --- a/openbb_platform/extensions/economy/integration/test_economy_python.py +++ b/openbb_platform/extensions/economy/integration/test_economy_python.py @@ -21,7 +21,6 @@ def obb(pytestconfig): # pylint: disable=inconsistent-return-statements @parametrize( "params", [ - ({"start_date": "2023-01-01", "end_date": "2023-06-06", "provider": "fmp"}), ( { "provider": "nasdaq", @@ -36,8 +35,16 @@ def obb(pytestconfig): # pylint: disable=inconsistent-return-statements "start_date": "2023-01-01", "end_date": "2023-06-06", "country": "mexico,sweden", - "importance": "Medium", + "importance": "low", "group": "gdp", + "calendar_id": None, + } + ), + ( + { + "provider": "fmp", + "start_date": "2023-10-24", + "end_date": "2023-11-03", } ), ], diff --git a/openbb_platform/openbb/assets/reference.json b/openbb_platform/openbb/assets/reference.json index 5ff26093f05..febb1baad07 100644 --- a/openbb_platform/openbb/assets/reference.json +++ b/openbb_platform/openbb/assets/reference.json @@ -2211,15 +2211,22 @@ }, { "name": "importance", - "type": "Literal['Low', 'Medium', 'High']", + "type": "Literal['low', 'medium', 'high']", "description": "Importance of the event.", "default": null, "optional": true }, { "name": "group", - "type": "Literal['interest rate', 'inflation', 'bonds', 'consumer', 'gdp', 'government', 'housing', 'labour', 'markets', 'money', 'prices', 'trade', 'business']", - "description": "Grouping of events", + "type": "Literal['interest_rate', 'inflation', 'bonds', 'consumer', 'gdp', 'government', 'housing', 'labour', 'markets', 'money', 'prices', 'trade', 'business']", + "description": "Grouping of events.", + "default": null, + "optional": true + }, + { + "name": "calendar_id", + "type": "Union[Union[int, str], List[Union[int, str]]]", + "description": "Get events by TradingEconomics Calendar ID. Multiple items allowed for provider(s): tradingeconomics.", "default": null, "optional": true } @@ -2271,6 +2278,13 @@ "optional": true }, { + "name": "category", + "type": "str", + "description": "Category of event.", + "default": null, + "optional": true + }, + { "name": "event", "type": "str", "description": "Event name.", @@ -2278,9 +2292,9 @@ "optional": true }, { - "name": "reference", + "name": "importance", "type": "str", - "description": "Abbreviated period for which released data refers to.", + "description": "The importance level for the event.", "default": null, "optional": true }, @@ -2292,16 +2306,23 @@ "optional": true }, { - "name": "sourceurl", + "name": "currency", "type": "str", - "description": "Source URL.", + "description": "Currency of the data.", "default": null, "optional": true }, { - "name": "actual", + "name": "unit", + "type": "str", + "description": "Unit of the data.", + "default": null, + "optional": true + }, + { + "name": "consensus", "type": "Union[str, float]", - "description": "Latest released value.", + "description": "Average forecast among a representative group of economists.", "default": null, "optional": true }, @@ -2313,79 +2334,122 @@ "optional": true }, { - "name": "consensus", + "name": "revised", "type": "Union[str, float]", - "description": "Average forecast among a representative group of economists.", + "description": "Revised previous value, if applicable.", "default": null, "optional": true }, { + "name": "actual", + "type": "Union[str, float]", + "description": "Latest released value.", + "default": null, + "optional": true + } + ], + "fmp": [ + { + "name": "change", + "type": "float", + "description": "Value change since previous.", + "default": null, + "optional": true + }, + { + "name": "change_percent", + "type": "float", + "description": "Percentage change since previous.", + "default": null, + "optional": true + }, + { + "name": "last_updated", + "type": "datetime", + "description": "Last updated timestamp.", + "default": null, + "optional": true + }, + { + "name": "created_at", + "type": "datetime", + "description": "Created at timestamp.", + "default": null, + "optional": true + } + ], + "tradingeconomics": [ + { "name": "forecast", "type": "Union[str, float]", - "description": "Trading Economics projections", + "description": "TradingEconomics projections.", "default": null, "optional": true }, { - "name": "url", + "name": "reference", "type": "str", - "description": "Trading Economics URL", + "description": "Abbreviated period for which released data refers to.", "default": null, "optional": true }, { - "name": "importance", - "type": "Union[Literal[0, 1, 2, 3], str]", - "description": "Importance of the event. 1-Low, 2-Medium, 3-High", + "name": "reference_date", + "type": "date", + "description": "Date for the reference period.", "default": null, "optional": true }, { - "name": "currency", - "type": "str", - "description": "Currency of the data.", + "name": "calendar_id", + "type": "int", + "description": "TradingEconomics Calendar ID.", "default": null, "optional": true }, { - "name": "unit", + "name": "date_span", + "type": "int", + "description": "Date span of the event.", + "default": null, + "optional": true + }, + { + "name": "symbol", "type": "str", - "description": "Unit of the data.", + "description": "TradingEconomics Symbol.", "default": null, "optional": true - } - ], - "fmp": [ + }, { - "name": "change", - "type": "float", - "description": "Value change since previous.", + "name": "ticker", + "type": "str", + "description": "TradingEconomics Ticker symbol.", "default": null, "optional": true }, { - "name": "change_percent", - "type": "float", - "description": "Percentage change since previous.", + "name": "te_url", + "type": "str", + "description": "TradingEconomics URL path.", "default": null, "optional": true }, { - "name": "updated_at", - "type": "datetime", - "description": "Last updated timestamp.", + "name": "source_url", + "type": "str", + "description": "Source URL.", "default": null, "optional": true }, { - "name": "created_at", + "name": "last_updated", "type": "datetime", - "description": "Created at timestamp.", + "description": "Last update of the data.", "default": null, "optional": true } - ], - "tradingeconomics": [] + ] }, "model": "EconomicCalendar" }, diff --git a/openbb_platform/openbb/package/economy.py b/openbb_platform/openbb/package/economy.py index dca7a2a489c..fcf6f22d4c8 100644 --- a/openbb_platform/openbb/package/economy.py +++ b/openbb_platform/openbb/package/economy.py @@ -156,10 +156,12 @@ class ROUTER_economy(Container): no default. country : Optional[str] Country of the event. Multiple comma separated items allowed. (provider: tradingeconomics) - importance : Optional[Literal['Low', 'Medium', 'High']] + importance : Optional[Literal['low', 'medium', 'high']] Importance of the event. (provider: tradingeconomics) - group : Optional[Literal['interest rate', 'inflation', 'bonds', 'consumer', 'gdp', 'government', 'housing', 'labour', 'markets', 'money', 'prices', 'trade', 'business']] - Grouping of events (provider: tradingeconomics) + group : Optional[Literal['interest_rate', 'inflation', 'bonds', 'consumer', 'gdp', 'government', 'housing', 'labour', 'markets', 'money', 'prices', 'trade', 'business']] + Grouping of events. (provider: tradingeconomics) + calendar_id : Optional[Union[int, str]] + Get events by TradingEconomics Calendar ID. Multiple comma separated items allowed. (provider: tradingeconomics) Returns ------- @@ -181,38 +183,53 @@ class ROUTER_economy(Container): The date of the data. country : Optional[str] Country of event. + category : Optional[str] + Category of event. event : Optional[str] Event name. - reference : Optional[str] - Abbreviated period for which released data refers to. + importance : Optional[str] + The importance level for the event. source : Optional[str] Source of the data. - sourceurl : Optional[str] - Source URL. - actual : Optional[Union[str, float]] - Latest released value. - previous : Optional[Union[str, float]] - Value for the previous period after the revision (if revision is applicable). - consensus : Optional[Union[str, float]] - Average forecast among a representative group of economists. - forecast : Optional[Union[str, float]] - Trading Economics projections - url : Optional[str] - Trading Economics URL - importance : Optional[Union[Literal[0, 1, 2, 3], str]] - Importance of the event. 1-Low, 2-Medium, 3-High currency : Optional[str] Currency of the data. unit : Optional[str] Unit of the data. + consensus : Optional[Union[str, float]] + Average forecast among a representative group of economists. + previous : Optional[Union[str, float]] + Value for the previous period after the revision (if revision is applicable). + revised : Optional[Union[str, float]] + Revised previous value, if applicable. + actual : Optional[Union[str, float]] + Latest released value. change : Optional[float] Value change since previous. (provider: fmp) change_percent : Optional[float] Percentage change since previous. (provider: fmp) - updated_at : Optional[datetime] - Last updated timestamp. (provider: fmp) + last_updated : Optional[datetime] + Last updated timestamp. (provider: fmp); + Last update of the data. (provider: tradingeconomics) created_at : Optional[datetime] Created at timestamp. (provider: fmp) + forecast : Optional[Union[str, float]] + TradingEconomics projections. (provider: tradingeconomics) + reference : Optional[str] + Abbreviated period for which released data refers to. (provider: tradingeconomics) + reference_date : Optional[date] + Date for the reference period. (provider: tradingeconomics) + calendar_id : Optional[int] + TradingEconomics Calendar ID. (provider: tradingeconomics) + date_span : Optional[int] + Date span of the event. (provider: tradingeconomics) + symbol : Optional[str] + TradingEconomics Symbol. (provider: tradingeconomics) + ticker : Optional[str] + TradingEconomics Ticker symbol. (provider: tradingeconomics) + te_url : Optional[str] + TradingEconomics URL path. (provider: tradingeconomics) + source_url : Optional[str] + Source URL. (provider: tradingeconomics) Examples -------- @@ -238,7 +255,10 @@ class ROUTER_economy(Container): }, extra_params=kwargs, info={ - "country": {"tradingeconomics": {"multiple_items_allowed": True}} + "country": {"tradingeconomics": {"multiple_items_allowed": True}}, + "calendar_id": { + "tradingeconomics": {"multiple_items_allowed": True} + }, }, ) ) diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/economic_calendar.py b/openbb_platform/providers/fmp/openbb_fmp/models/economic_calendar.py index ba7b3306548..135aaa04314 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/economic_calendar.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/economic_calendar.py @@ -1,7 +1,11 @@ """FMP Economic Calendar Model.""" -from datetime import datetime +# pylint: disable=unused-argument + +import asyncio +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional +from warnings import warn from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.standard_models.economic_calendar import ( @@ -9,7 +13,7 @@ from openbb_core.provider.standard_models.economic_calendar import ( EconomicCalendarQueryParams, ) from openbb_core.provider.utils.helpers import amake_request -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator class FMPEconomicCalendarQueryParams(EconomicCalendarQueryParams): @@ -25,7 +29,13 @@ class FMPEconomicCalendarData(EconomicCalendarData): Source: https://site.financialmodelingprep.com/developer/docs/economic-calendar-api """ - __alias_dict__ = {"consensus": "estimate", "importance": "impact"} + __alias_dict__ = { + "consensus": "estimate", + "importance": "impact", + "last_updated": "updatedAt", + "created_at": "createdAt", + "change_percent": "changePercentage", + } change: Optional[float] = Field( description="Value change since previous.", @@ -34,29 +44,31 @@ class FMPEconomicCalendarData(EconomicCalendarData): change_percent: Optional[float] = Field( description="Percentage change since previous.", default=None, - alias="changePercentage", ) - updated_at: Optional[datetime] = Field( - description="Last updated timestamp.", default=None, alias="updatedAt" + last_updated: Optional[datetime] = Field( + description="Last updated timestamp.", default=None ) created_at: Optional[datetime] = Field( - description="Created at timestamp.", default=None, alias="createdAt" + description="Created at timestamp.", default=None ) - @field_validator("date", mode="before", check_fields=False) - def date_validate(cls, v: str): # pylint: disable=E0213 - """Return the date as a datetime object.""" - return datetime.strptime(v, "%Y-%m-%d %H:%M:%S") if v else None - - @field_validator("updatedAt", mode="before", check_fields=False) - def updated_at_validate(cls, v: str): # pylint: disable=E0213 + @field_validator( + "date", "last_updated", "created_at", mode="before", check_fields=False + ) + @classmethod + def date_validate(cls, v: str): """Return the date as a datetime object.""" return datetime.strptime(v, "%Y-%m-%d %H:%M:%S") if v else None - @field_validator("createdAt", mode="before", check_fields=False) - def created_at_validate(cls, v: str): # pylint: disable=E0213 - """Return the date ending as a datetime object.""" - return datetime.strptime(v, "%Y-%m-%d %H:%M:%S") if v else None + @model_validator(mode="before") + @classmethod + def empty_strings(cls, values): + """Replace empty values with None.""" + return ( + {k: (None if v in ("", 0) else v) for k, v in values.items()} + if isinstance(values, dict) + else values + ) class FMPEconomicCalendarFetcher( @@ -70,13 +82,12 @@ class FMPEconomicCalendarFetcher( @staticmethod def transform_query(params: Dict[str, Any]) -> FMPEconomicCalendarQueryParams: """Transform the query.""" - if params: - if params["start_date"] is None: - params["start_date"] = datetime.now().strftime("%Y-%m-%d") - if params["end_date"] is None: - params["end_date"] = datetime.now().strftime("%Y-%m-%d") - - return FMPEconomicCalendarQueryParams(**params) + transformed_params = params + if not transformed_params.get("start_date"): + transformed_params["start_date"] = datetime.now().date() + if not transformed_params.get("end_date"): + transformed_params["end_date"] = (datetime.now() + timedelta(days=7)).date() + return FMPEconomicCalendarQueryParams(**transformed_params) @staticmethod async def aextract_data( @@ -89,11 +100,41 @@ class FMPEconomicCalendarFetcher( base_url = "https://financialmodelingprep.com/api/v3/economic_calendar?" - url = f"{base_url}from={query.start_date}&to={query.end_date}&apikey={api_key}" - - return await amake_request(url, **kwargs) + # FMP allows only 3-month windows to be queried, we need to chunk to request. + def date_range(start_date, end_date): + """Yield start and end dates for each 90-day period between start_date and end_date.""" + delta = timedelta(days=90) + current_date = start_date + while current_date < end_date: + next_date = min(current_date + delta, end_date) + yield current_date, next_date + current_date = next_date + timedelta(days=1) + + date_ranges = list(date_range(query.start_date, query.end_date)) + urls = [ + f"{base_url}from={start_date.strftime('%Y-%m-%d')}&to={end_date.strftime('%Y-%m-%d')}&apikey={api_key}" + for start_date, end_date in date_ranges + ] + results: List[Dict] = [] + + # We need to do this because Pytest does not seem to be able to handle `amake_requests`. + async def get_one(url): + """Get data for one URL.""" + n_urls = 1 + try: + result = await amake_request(url, **kwargs) + if result: + results.extend(result) + except Exception as e: + if len(urls) == 1 or (len(urls) > 1 and n_urls == len(urls)): + raise e from e + warn(f"Error in fetching part of the data from FMP -> {e}") + n_urls += 1 + + await asyncio.gather(*[get_one(url) for url in urls]) + + return results - # pylint: disable=unused-argument @staticmethod def transform_data( query: FMPEconomicCalendarQueryParams, diff --git a/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_economic_calendar_fetcher.yaml b/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_economic_calendar_fetcher.yaml index ecc1f2aa937..c5f7c0682dc 100644 --- a/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_economic_calendar_fetcher.yaml +++ b/openbb_platform/providers/fmp/tests/record/http/test_fmp_fetchers/test_fmp_economic_calendar_fetcher.yaml @@ -3,221 +3,2622 @@ interactions: body: null headers: Accept: - - '*/*' + - application/json |