diff options
author | Danglewood <85772166+deeleeramone@users.noreply.github.com> | 2024-05-07 13:03:13 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-07 20:03:13 +0000 |
commit | 99b0bb5287621d040f863090d5c7861f08809472 (patch) | |
tree | ef5d8e58a71e294b3f43d25c181ac45e7de850b3 | |
parent | e12aac157eb69573641aadcd93b1f244dd7dd6fe (diff) |
[Feature] EconDB Main Indicators (#6365)
* add main indicators to economy.indicators
* static assets
* ruff
* Adapt and add unit test
* record test cassette
* polygon test cassette
* currency pairs
* recapture test
* mypy
---------
Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
12 files changed, 1476 insertions, 428 deletions
diff --git a/openbb_platform/core/openbb_core/provider/standard_models/economic_indicators.py b/openbb_platform/core/openbb_core/provider/standard_models/economic_indicators.py index 3abbfc861ea..013c962440a 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/economic_indicators.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/economic_indicators.py @@ -3,7 +3,7 @@ from datetime import date as dateType from typing import Optional, Union -from pydantic import Field, field_validator +from pydantic import Field from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.query_params import QueryParams @@ -32,12 +32,6 @@ class EconomicIndicatorsQueryParams(QueryParams): description=QUERY_DESCRIPTIONS.get("end_date", ""), default=None ) - @field_validator("symbol", mode="before", check_fields=False) - @classmethod - def to_upper(cls, v: str) -> str: - """Convert field to uppercase.""" - return v.upper() - class EconomicIndicatorsData(Data): """Economic Indicators Data.""" diff --git a/openbb_platform/extensions/economy/integration/test_economy_api.py b/openbb_platform/extensions/economy/integration/test_economy_api.py index 839065cecef..54ea5f6148f 100644 --- a/openbb_platform/extensions/economy/integration/test_economy_api.py +++ b/openbb_platform/extensions/economy/integration/test_economy_api.py @@ -576,6 +576,19 @@ def test_economy_fred_regional(params, headers): "start_date": "2022-01-01", "end_date": "2024-01-01", "use_cache": False, + "frequency": None, + } + ), + ( + { + "provider": "econdb", + "country": None, + "symbol": "MAIN", + "transform": None, + "start_date": "2022-01-01", + "end_date": "2024-01-01", + "use_cache": False, + "frequency": "quarter", } ), ], diff --git a/openbb_platform/extensions/economy/integration/test_economy_python.py b/openbb_platform/extensions/economy/integration/test_economy_python.py index a838c71a272..9a9eb1e35bd 100644 --- a/openbb_platform/extensions/economy/integration/test_economy_python.py +++ b/openbb_platform/extensions/economy/integration/test_economy_python.py @@ -565,6 +565,19 @@ def test_economy_available_indicators(params, obb): "start_date": "2022-01-01", "end_date": "2024-01-01", "use_cache": False, + "frequency": None, + } + ), + ( + { + "provider": "econdb", + "country": None, + "symbol": "MAIN", + "transform": None, + "start_date": "2022-01-01", + "end_date": "2024-01-01", + "use_cache": False, + "frequency": "quarter", } ), ], diff --git a/openbb_platform/extensions/economy/openbb_economy/economy_router.py b/openbb_platform/extensions/economy/openbb_economy/economy_router.py index 7ccb4a723d1..19eb403cea1 100644 --- a/openbb_platform/extensions/economy/openbb_economy/economy_router.py +++ b/openbb_platform/extensions/economy/openbb_economy/economy_router.py @@ -362,6 +362,10 @@ async def available_indicators( "provider": "econdb", }, ), + APIEx( + description="Use the `main` symbol to get the group of main indicators for a country.", + parameters={"provider": "econdb", "symbol": "main", "country": "eu"}, + ), ], ) async def indicators( diff --git a/openbb_platform/openbb/assets/reference.json b/openbb_platform/openbb/assets/reference.json index 94947e5c4eb..004e047344c 100644 --- a/openbb_platform/openbb/assets/reference.json +++ b/openbb_platform/openbb/assets/reference.json @@ -4108,7 +4108,7 @@ "message": null }, "description": "Get economic indicators by country and indicator.", - "examples": "\nExamples\n--------\n\n```python\nfrom openbb import obb\nobb.economy.indicators(provider='econdb', symbol='PCOCO')\n# Enter the country as the full name, or iso code. Use `available_indicators()` to get a list of supported indicators from EconDB.\nobb.economy.indicators(symbol='CPI', country='united_states,jp', provider='econdb')\n```\n\n", + "examples": "\nExamples\n--------\n\n```python\nfrom openbb import obb\nobb.economy.indicators(provider='econdb', symbol='PCOCO')\n# Enter the country as the full name, or iso code. Use `available_indicators()` to get a list of supported indicators from EconDB.\nobb.economy.indicators(symbol='CPI', country='united_states,jp', provider='econdb')\n# Use the `main` symbol to get the group of main indicators for a country.\nobb.economy.indicators(provider='econdb', symbol='main', country='eu')\n```\n\n", "parameters": { "standard": [ { @@ -4151,14 +4151,21 @@ { "name": "transform", "type": "Literal['toya', 'tpop', 'tusd', 'tpgp']", - "description": "The transformation to apply to the data, default is None. tpop: Change from previous period toya: Change from one year ago tusd: Values as US dollars tpgp: Values as a percent of GDP Only 'tpop' and 'toya' are applicable to all indicators. Applying transformations across multiple indicators/countries may produce unexpected results. This is because not all indicators are compatible with all transformations, and the original units and scale differ between entities. `tusd` should only be used where values are currencies.", + "description": "The transformation to apply to the data, default is None. tpop: Change from previous period toya: Change from one year ago tusd: Values as US dollars tpgp: Values as a percent of GDP Only 'tpop' and 'toya' are applicable to all indicators. Applying transformations across multiple indicators/countries may produce unexpected results. This is because not all indicators are compatible with all transformations, and the original units and scale differ between entities. `tusd` should only be used where values are currencies.", "default": null, "optional": true }, { + "name": "frequency", + "type": "Literal['annual', 'quarter', 'month']", + "description": "The frequency of the data, default is 'quarter'. Only valid when 'symbol' is 'main'.", + "default": "quarter", + "optional": true + }, + { "name": "use_cache", "type": "bool", - "description": "If True, the request will be cached for one day.Using cache is recommended to avoid needlessly requesting the same data.", + "description": "If True, the request will be cached for one day. Using cache is recommended to avoid needlessly requesting the same data.", "default": true, "optional": true } diff --git a/openbb_platform/openbb/package/economy.py b/openbb_platform/openbb/package/economy.py index 0422146ba39..56deffca0aa 100644 --- a/openbb_platform/openbb/package/economy.py +++ b/openbb_platform/openbb/package/economy.py @@ -1083,19 +1083,21 @@ class ROUTER_economy(Container): The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'econdb' if there is no default. - transform : Literal['toya', 'tpop', 'tusd', 'tpgp'] + transform : Optional[Literal['toya', 'tpop', 'tusd', 'tpgp']] The transformation to apply to the data, default is None. - tpop: Change from previous period - toya: Change from one year ago - tusd: Values as US dollars - tpgp: Values as a percent of GDP + tpop: Change from previous period + toya: Change from one year ago + tusd: Values as US dollars + tpgp: Values as a percent of GDP - Only 'tpop' and 'toya' are applicable to all indicators. Applying transformations across multiple indicators/countries may produce unexpected results. - This is because not all indicators are compatible with all transformations, and the original units and scale differ between entities. - `tusd` should only be used where values are currencies. (provider: econdb) + Only 'tpop' and 'toya' are applicable to all indicators. Applying transformations across multiple indicators/countries may produce unexpected results. + This is because not all indicators are compatible with all transformations, and the original units and scale differ between entities. + `tusd` should only be used where values are currencies. (provider: econdb) + frequency : Literal['annual', 'quarter', 'month'] + The frequency of the data, default is 'quarter'. Only valid when 'symbol' is 'main'. (provider: econdb) use_cache : bool - If True, the request will be cached for one day.Using cache is recommended to avoid needlessly requesting the same data. (provider: econdb) + If True, the request will be cached for one day. Using cache is recommended to avoid needlessly requesting the same data. (provider: econdb) Returns ------- @@ -1130,6 +1132,8 @@ class ROUTER_economy(Container): >>> obb.economy.indicators(provider='econdb', symbol='PCOCO') >>> # Enter the country as the full name, or iso code. Use `available_indicators()` to get a list of supported indicators from EconDB. >>> obb.economy.indicators(symbol='CPI', country='united_states,jp', provider='econdb') + >>> # Use the `main` symbol to get the group of main indicators for a country. + >>> obb.economy.indicators(provider='econdb', symbol='main', country='eu') """ # noqa: E501 return self._run( diff --git a/openbb_platform/providers/econdb/openbb_econdb/models/economic_indicators.py b/openbb_platform/providers/econdb/openbb_econdb/models/economic_indicators.py index ffbadbc0a67..3bea1e58cc6 100644 --- a/openbb_platform/providers/econdb/openbb_econdb/models/economic_indicators.py +++ b/openbb_platform/providers/econdb/openbb_econdb/models/economic_indicators.py @@ -3,7 +3,7 @@ # pylint: disable=unused-argument from datetime import datetime, timedelta -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from warnings import warn from openbb_core.provider.abstract.annotated_result import AnnotatedResult @@ -14,6 +14,7 @@ from openbb_core.provider.standard_models.economic_indicators import ( ) from openbb_core.provider.utils.errors import EmptyDataError from openbb_econdb.utils import helpers +from openbb_econdb.utils.main_indicators import get_main_indicators from pandas import DataFrame, concat from pydantic import Field, field_validator @@ -28,27 +29,32 @@ class EconDbEconomicIndicatorsQueryParams(EconomicIndicatorsQueryParams): "country": ["multiple_items_allowed"], } - transform: Literal["toya", "tpop", "tusd", "tpgp"] = Field( + transform: Union[None, Literal["toya", "tpop", "tusd", "tpgp"]] = Field( default=None, description="The transformation to apply to the data, default is None." + "\n" - + "\n tpop: Change from previous period" - + "\n toya: Change from one year ago" - + "\n tusd: Values as US dollars" - + "\n tpgp: Values as a percent of GDP" + + "\n tpop: Change from previous period" + + "\n toya: Change from one year ago" + + "\n tusd: Values as US dollars" + + "\n tpgp: Values as a percent of GDP" + "\n" + "\n" - + " Only 'tpop' and 'toya' are applicable to all indicators." - + " Applying transformations across multiple indicators/countries" - + " may produce unexpected results." - + "\n This is because not all indicators are compatible with all transformations," - + " and the original units and scale differ between entities." - + "\n `tusd` should only be used where values are currencies.", + + " Only 'tpop' and 'toya' are applicable to all indicators." + + " Applying transformations across multiple indicators/countries" + + " may produce unexpected results." + + "\n This is because not all indicators are compatible with all transformations," + + " and the original units and scale differ between entities." + + "\n `tusd` should only be used where values are currencies.", + ) + frequency: Literal["annual", "quarter", "month"] = Field( + default="quarter", + description="The frequency of the data, default is 'quarter'." + + " Only valid when 'symbol' is 'main'.", ) use_cache: bool = Field( default=True, description="If True, the request will be cached for one day." - + "Using cache is recommended to avoid needlessly requesting the same data.", + + " Using cache is recommended to avoid needlessly requesting the same data.", ) @field_validator("country", mode="before", check_fields=False) @@ -89,12 +95,20 @@ class EconDbEconomicIndicatorsQueryParams(EconomicIndicatorsQueryParams): @classmethod def validate_symbols(cls, v): """Validate each symbol to check if it is a valid indicator.""" + if not v: + v = "main" symbols = v if isinstance(v, list) else v.split(",") new_symbols: List[str] = [] for symbol in symbols: if "_" in symbol: new_symbols.append(symbol) continue + if symbol.upper() == "MAIN": + if len(symbols) > 1: + raise ValueError( + "The 'main' indicator cannot be combined with other indicators." + ) + return symbol if not any( ( symbol.upper().startswith(indicator) @@ -135,6 +149,15 @@ class EconDbEconomicIndicatorsFetcher( ).date() if new_params.get("end_date") is None: new_params["end_date"] = datetime.today().date() + countries = new_params.get("country") + if ( + countries is not None + and len(countries.split(",")) > 1 + and new_params.get("symbol", "").upper() == "MAIN" + ): + raise ValueError( + "The 'main' indicator cannot be combined with multiple countries." + ) return EconDbEconomicIndicatorsQueryParams(**new_params) @staticmethod @@ -144,6 +167,16 @@ class EconDbEconomicIndicatorsFetcher( **kwargs: Any, ) -> List[Dict]: """Extract the data.""" + if query.symbol.upper() == "MAIN": + country = query.country.upper() if query.country else "US" + return await get_main_indicators( + country, + query.start_date.strftime("%Y-%m-%d"), # type: ignore + query.end_date.strftime("%Y-%m-%d"), # type: ignore + query.frequency, + query.transform, + query.use_cache, + ) token = credentials.get("econdb_api_key", "") # type: ignore # Attempt to create a temporary token if one is not supplied. if not token: @@ -299,6 +332,14 @@ class EconDbEconomicIndicatorsFetcher( **kwargs: Any, ) -> AnnotatedResult[List[EconDbEconomicIndicatorsData]]: """Transform the data.""" + if query.symbol.upper() == "MAIN": + return AnnotatedResult( + result=[ + EconDbEconomicIndicatorsData.model_validate(r) + for r in data[0].get("records", []) + ], + metadata={query.country: data[0].get("metadata", [])}, + ) output = DataFrame() metadata = {} for d in data: diff --git a/openbb_platform/providers/econdb/openbb_econdb/utils/main_indicators.py b/openbb_platform/providers/econdb/openbb_econdb/utils/main_indicators.py new file mode 100644 index 00000000000..152fa876713 --- /dev/null +++ b/openbb_platform/providers/econdb/openbb_econdb/utils/main_indicators.py @@ -0,0 +1,227 @@ +"""Main Indicators""" + +from datetime import datetime, timedelta +from typing import Dict, List, Literal + +from aiohttp_client_cache import SQLiteBackend +from aiohttp_client_cache.session import CachedSession +from numpy import arange +from openbb_core.app.utils import get_user_cache_directory +from openbb_core.provider.utils.helpers import amake_request +from openbb_econdb.utils.helpers import COUNTRY_MAP, THREE_LETTER_ISO_MAP +from pandas import Categorical, DataFrame, Series, concat, to_datetime + +trends_transform_labels_dict = { + 1: "Change from previous period.", + 2: "Change from one year ago.", + 3: "Level", + 9: "Level (USD)", +} +trends_freq_dict = { + "annual": "Y", + "quarter": "Q", + "month": "M", +} +trends_transform_dict = { + "tpop": 1, + "toya": 2, + "level": 3, + "tusd": 9, + None: 3, +} + +main_indicators_order = [ + "RGDP", + "RPRC", + "RPUC", + "RGFCF", + "REXP", + "RIMP", + "GDP", + "PRC", + "PUC", + "GFCF", + "EXP", + "IMP", + "CPI", + "PPI", + "CORE", + "URATE", + "EMP", + "ACPOP", + "RETA", + "CONF", + "IP", + "CP", + "GBAL", + "GREV", + "GSPE", + "GDEBT", + "CA", + "TB", + "NIIP", + "IIPA", + "IIPL", + "Y10YD", + "M3YD", + "HOU", + "OILPROD", + "POP", +] + + +async def fetch_data(url, use_cache: bool = True): + """Fetch the data with or without the cached session object.""" + if use_cache is True: + cache_dir = f"{get_user_cache_directory()}/http/econdb_main_indicators" + async with CachedSession( + cache=SQLiteBackend(cache_dir, expire_after=3600 * 24) + ) as session: + try: + response = await amake_request(url, session=session) + finally: + await session.close() + else: + response = await amake_request(url) + + return response + + +async def get_main_indicators( # pylint: disable=R0913,R0914,R0915 + country: str = "US", + start_date: str = (datetime.now() - timedelta(weeks=52 * 3)).strftime("%Y-%m-%d"), + end_date: str = datetime.now().strftime("%Y-%m-%d"), + frequency: Literal["annual", "quarter", "month"] = "quarter", + transform: Literal["tpop", "toya", "level", "tusd", None] = "toya", + use_cache: bool = True, +) -> List[Dict]: + """Get the main indicators for a given country.""" + freq = trends_freq_dict.get(frequency) + transform = trends_transform_dict.get(transform) # type: ignore + if len(country) == 3: + country = THREE_LETTER_ISO_MAP.get(country.upper()) + if not country: + raise ValueError(f"Error: Invalid country code -> {country}") + if country in COUNTRY_MAP: + country = COUNTRY_MAP.get(country) + if len(country) != 2: + raise ValueError( + f"Error: Please supply a 2-Letter ISO Country Code -> {country}" + ) + if country not in COUNTRY_MAP.values(): + raise ValueError(f"Error: Invalid country code -> {country}") + parents_url = ( + "https://www.econdb.com/trends/country_forecast/" + + f"?country={country}&freq={freq}&transform={transform}" + + f"&dateStart={start_date}&dateEnd={end_date}" + ) + r = await fetch_data(parents_url, use_cache) + row_names = r.get("row_names") + row_symbols = [] + row_is_parent = [] + row_symbols = [d["code"] for d in row_names] + row_is_parent = [d["is_parent"] for d in row_names] + parent_map = {d["code"]: d["is_parent"] for d in row_names} + units_col = r.get("units_col") + metadata = r.get("footnote") + row_names = r.get("row_names") + row_name_map = {d["code"]: d["verbose"].title() for d in row_names} + row_units_dict = dict(zip(row_symbols, units_col)) + units_df = concat([Series(units_col), Series(row_is_parent)], axis=1) + units_df.columns = ["units", "is_parent"] + df = DataFrame(r["data"]).set_index("indicator") + df = df.pivot(columns="obs_time", values="obs_value").filter( + items=row_symbols, axis=0 + ) + df["units"] = df.index.map(row_units_dict.get) + df["is_parent"] = df.index.map(parent_map.get) + df = df.set_index("is_parent", append=True) + + async def get_children( # pylint: disable=R0913 + parent, country, freq, transform, start_date, end_date, use_cache + ) -> DataFrame: + """Get the child elements for the main indicator symbols.""" + children_url = ( + "https://www.econdb.com/trends/get_topic_children/" + + f"?country={country}&agency=3&freq={freq}&transform={transform}" + + f"&parent_id={parent}&dateStart={start_date}&dateEnd={end_date}" + ) + child_r = await fetch_data(children_url, use_cache) + row_names = child_r.get("row_names") + row_symbols = [] + row_symbols = [d["code"] for d in row_names] + units_col = child_r.get("units_col") + metadata.extend(child_r.get("footnote")) + row_names = child_r.get("row_names") + row_name_map.update({d["code"]: d["verbose"].title() for d in row_names}) + row_units_dict = dict(zip(row_symbols, units_col)) + child_df = DataFrame(child_r["data"]).set_index("indicator") + child_df = child_df.pivot(columns="obs_time", values="obs_value").filter( + items=row_symbols, axis=0 + ) + child_df["units"] = child_df.index.map(row_units_dict.get) + # Set 'units' to 'Index' when the index is 'CONF' + if "CONF" in child_df.index and child_df.loc["CONF", "units"] == "..": + child_df.loc["CONF", "units"] = "Index" + child_df["is_parent"] = parent + child_df = child_df.reset_index() + child_df["name"] = child_df["indicator"].map(row_name_map) + return child_df + + new_df = df.reset_index() + has_children = new_df[ + new_df["is_parent"] == True # noqa pylint: disable=C0121 + ].indicator.to_list() + + async def append_children( # pylint: disable=R0913 + df, parent, country, freq, transform, start_date, end_date, use_cache + ): + """Get the child element and insert it below the parent row.""" + temp = DataFrame() + try: + children = await get_children( + parent, country, freq, transform, start_date, end_date, use_cache + ) + except Exception as _: # pylint: disable=W0718 + return df + idx = df[df["indicator"] == parent].index[0] + df1 = df[df.index <= idx] + df2 = df[df.index > idx] + temp = concat([df1, children, df2]) + return temp + + # Get the child elements for each parent. + for parent in has_children: + new_df = await append_children( + new_df, parent, country, freq, transform, start_date, end_date, use_cache + ) + + # Cast the shape, specify the order and flatten for output. + new_df["name"] = new_df["indicator"].map(row_name_map) + new_df.set_index(["indicator", "is_parent", "name", "units"], inplace=True) + new_df.columns = new_df.columns + new_df.columns = [to_datetime(d).strftime("%Y-%m-%d") for d in new_df.columns] + for col in new_df.columns: + new_df[col] = new_df[col].astype(str).str.replace(" ", "").astype(float) + new_df = new_df.apply(lambda row: row / 100 if "%" in row.name[3] else row, axis=1) + new_df = new_df.iloc[:, ::-1] + new_df = new_df.fillna("N/A").replace("N/A", None) + output = new_df + output.columns.name = "date" + output = output.reset_index() + filtered_df = output[output["indicator"].isin(main_indicators_order)].copy() + filtered_df["indicator"] = Categorical( + filtered_df["indicator"], categories=main_indicators_order, ordered=True + ) + filtered_df.sort_values("indicator", inplace=True) + output = filtered_df + output.set_index(["indicator", "is_parent", "name", "units"], inplace=True) + output["index_order"] = arange(len(output)) + output = output.reset_index().melt( + id_vars=["index_order", "indicator", "is_parent", "name", "units"], + var_name="date", + value_name="value", + ) + output = output.rename(columns={"indicator": "symbol_root"}) + results = {"records": output.to_dict(orient="records"), "metadata": metadata} + return [results] diff --git a/openbb_platform/providers/econdb/tests/record/http/test_econdb_fetchers/test_econdb_economic_indicators_main_fetcher.yaml b/openbb_platform/providers/econdb/tests/record/http/test_econdb_fetchers/test_econdb_economic_indicators_main_fetcher.yaml new file mode 100644 index 00000000000..b153e357abd --- /dev/null +++ b/openbb_platform/providers/econdb/tests/record/http/test_econdb_fetchers/test_econdb_economic_indicators_main_fetcher.yaml @@ -0,0 +1,725 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: https://www.econdb.com/trends/country_forecast/?country=JP&dateEnd=2024-04-01&dateStart=2022-01-01&freq=Q&transform=3 + response: + body: + string: !!binary | + H4sIAAAAAAAAA+1cW2/buBL+K4SBvkWObr7tW3Np1kHapEn60BMUhmIzqc7Kko8ktzUW/e9LDiVb + jmUPa8rM4pRAWtgSOfxMDb8ZzlDzd2sS5EHrD/Lwdyt5zEZ5OKXsW8u1XdeyHfZ3b9t/wN9/WkcE + 2nwLojk0crwecQYDfj3MRk9JSsdBlrM7T0GUUX41noTjIE9S3vr24uyGNx0n8zhPF/zSJVwInmk8 + XozCCbsUz6Po5xGpA+NjYHziubYmMD0MTIcMbF8PGMdGwPg2GXRcHWA8XGcYmJ6vZWY8CZ3pEd/2 + NIFBdaZPGCA9YHCdcUlXDxgZnukTu+tIgjk0zXAsshNzeJbpeR0tWHCF6ZG+r2NeZDjGJ31PlvAO + TDF+l/g9TfOC6YvfIY6rYx1JEEynQ/yOpjW9W18c22nbcjhOb4aH4xbHdjXh2K0nju3pwYHoiGP7 + OnCgfKIPB6YfHU04MP3o6sGB6kdPBw4f1w8tOHA/yW13Jfn00+3b+/MDekluW9biKSNBbJ6+OcEs + ntuW3fWoIcH9I11zgntHuvQE9430zcm/RU9QXtPJJxjDSnskt+f3bw/pokm7AqpAMBss7QsoAkGN + sLQzoAREwksb6AKC6Ihj6wKC6IgjvZ9QBILpiCO9wVJdNZiOSK+a4QGDSL/AImowMA6R9luVYKAM + ouOhN |