diff options
author | Pratyush Shukla <ps4534@nyu.edu> | 2024-02-02 16:42:50 +0530 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-02 11:12:50 +0000 |
commit | 9ddbf34456b748fd38aa7e1bd4e95618dc253f94 (patch) | |
tree | 075d343527ab81b9c65156b87b464cd6d2503a41 | |
parent | 5f8fca5a4a8896a87d5e7f5244627a21f53e3df9 (diff) |
[Feature] Add support for multiple tags in `historical_attributes` and `latest_attributes` (#6013)
* historical attributes standard model -
* add support multiple tags
* add `tag` field in data
* add support for multiple tags in historical attributes
* set correct return type in `callback` function
* latest attributes standard model -
* add support for multiple tags
* add `tag` field in data
* add support for multiple tags in latest attributes
* add test params for the attributes endpoints
* add support for multiple `symbol` and add `symbol` field in the data
* modified code to fetch data from multiple symbols and throw warning as per @deeleeramone
* add params for testing multiple symbols
* linting
* fix test params
* fixed condition to check if its a `Dict` type
* updated intrinio fetcher tests for the statements
* updated static
8 files changed, 332 insertions, 33 deletions
diff --git a/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py b/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py index 8bda172520d..54fa6bda03d 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py @@ -1,9 +1,9 @@ """Historical Attributes Standard Model.""" from datetime import date as dateType -from typing import Literal, Optional +from typing import List, Literal, Optional, Set, Union -from pydantic import Field +from pydantic import Field, field_validator from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.query_params import QueryParams @@ -30,16 +30,36 @@ class HistoricalAttributesQueryParams(QueryParams): limit: Optional[int] = Field( default=1000, description=QUERY_DESCRIPTIONS.get("limit") ) - type: Optional[str] = Field( + tag_type: Optional[str] = Field( default=None, description="Filter by type, when applicable." ) sort: Optional[Literal["asc", "desc"]] = Field( default="desc", description="Sort order." ) + @field_validator("tag", mode="before", check_fields=False) + @classmethod + def multiple_tags(cls, v: Union[str, List[str], Set[str]]): + """Accept a comma-separated string or list of tags.""" + if isinstance(v, str): + return v.lower() + return ",".join([tag.lower() for tag in list(v)]) + + @field_validator("symbol", mode="before", check_fields=False) + @classmethod + def upper_symbol(cls, v: Union[str, List[str], Set[str]]): + """Convert symbol to uppercase.""" + if isinstance(v, str): + return v.upper() + return ",".join([symbol.upper() for symbol in list(v)]) + class HistoricalAttributesData(Data): """Historical Attributes Data.""" date: dateType = Field(description=DATA_DESCRIPTIONS.get("date")) + symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol")) + tag: Optional[str] = Field( + default=None, description="Tag name for the fetched data." + ) value: Optional[float] = Field(default=None, description="The value of the data.") diff --git a/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py b/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py index d4245aa7a7a..e0a8744e516 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py @@ -1,12 +1,15 @@ """Latest Attributes Standard Model.""" -from typing import Optional, Union +from typing import List, Optional, Set, Union -from pydantic import Field +from pydantic import Field, field_validator from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.query_params import QueryParams -from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS +from openbb_core.provider.utils.descriptions import ( + DATA_DESCRIPTIONS, + QUERY_DESCRIPTIONS, +) class LatestAttributesQueryParams(QueryParams): @@ -15,10 +18,30 @@ class LatestAttributesQueryParams(QueryParams): symbol: str = Field(description=QUERY_DESCRIPTIONS.get("symbol")) tag: str = Field(description="Intrinio data tag ID or code.") + @field_validator("tag", mode="before", check_fields=False) + @classmethod + def multiple_tags(cls, v: Union[str, List[str], Set[str]]): + """Accept a comma-separated string or list of tags.""" + if isinstance(v, str): + return v.lower() + return ",".join([tag.lower() for tag in list(v)]) + + @field_validator("symbol", mode="before", check_fields=False) + @classmethod + def upper_symbol(cls, v: Union[str, List[str], Set[str]]): + """Convert symbol to uppercase.""" + if isinstance(v, str): + return v.upper() + return ",".join([symbol.upper() for symbol in list(v)]) + class LatestAttributesData(Data): """Latest Attributes Data.""" + symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol")) + tag: Optional[str] = Field( + default=None, description="Tag name for the fetched data." + ) value: Optional[Union[str, float]] = Field( default=None, description="The value of the data." ) diff --git a/openbb_platform/extensions/equity/integration/test_equity_api.py b/openbb_platform/extensions/equity/integration/test_equity_api.py index ca4b536fc08..6cd0709dcb1 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_api.py +++ b/openbb_platform/extensions/equity/integration/test_equity_api.py @@ -1055,7 +1055,59 @@ def test_equity_fundamental_search_attributes(params, headers): "tag": "ebit", "frequency": "yearly", "limit": 1000, - "type": None, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": "ebit,ebitda,marketcap", + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": ["ebit", "ebitda", "marketcap"], + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL,MSFT", + "tag": "ebit,ebitda,marketcap", + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": ["AAPL", "MSFT"], + "tag": ["ebit", "ebitda", "marketcap"], + "frequency": "yearly", + "limit": 1000, + "tag_type": None, "start_date": "2013-01-01", "end_date": "2023-01-01", "sort": "desc", @@ -1087,10 +1139,38 @@ def test_equity_fundamental_historical_attributes(params, headers): ( { "provider": "intrinio", - "symbol": "MSFT", + "symbol": "AAPL", "tag": "ebitda", } ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": "ceo,ebitda", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": ["ceo", "ebitda"], + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL,MSFT", + "tag": ["ceo", "ebitda"], + } + ), + ( + { + "provider": "intrinio", + "symbol": ["AAPL", "MSFT"], + "tag": ["ceo", "ebitda"], + } + ), ], ) @pytest.mark.integration diff --git a/openbb_platform/extensions/equity/integration/test_equity_python.py b/openbb_platform/extensions/equity/integration/test_equity_python.py index 914f18f9773..09adb9bb956 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_python.py +++ b/openbb_platform/extensions/equity/integration/test_equity_python.py @@ -999,7 +999,59 @@ def test_equity_fundamental_search_attributes(params, obb): "tag": "ebit", "frequency": "yearly", "limit": 1000, - "type": None, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": "ebit,ebitda,marketcap", + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": ["ebit", "ebitda", "marketcap"], + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL,MSFT", + "tag": "ebit,ebitda,marketcap", + "frequency": "yearly", + "limit": 1000, + "tag_type": None, + "start_date": "2013-01-01", + "end_date": "2023-01-01", + "sort": "desc", + } + ), + ( + { + "provider": "intrinio", + "symbol": ["AAPL", "MSFT"], + "tag": ["ebit", "ebitda", "marketcap"], + "frequency": "yearly", + "limit": 1000, + "tag_type": None, "start_date": "2013-01-01", "end_date": "2023-01-01", "sort": "desc", @@ -1020,6 +1072,7 @@ def test_equity_fundamental_historical_attributes(params, obb): [ ( { + "provider": "intrinio", "symbol": "AAPL", "tag": "ceo", } @@ -1028,14 +1081,35 @@ def test_equity_fundamental_historical_attributes(params, obb): { "provider": "intrinio", "symbol": "AAPL", - "tag": "ceo", + "tag": "ebitda", } ), ( { "provider": "intrinio", - "symbol": "MSFT", - "tag": "ebitda", + "symbol": "AAPL", + "tag": "ceo,ebitda", + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL", + "tag": ["ceo", "ebitda"], + } + ), + ( + { + "provider": "intrinio", + "symbol": "AAPL,MSFT", + "tag": ["ceo", "ebitda"], + } + ), + ( + { + "provider": "intrinio", + "symbol": ["AAPL", "MSFT"], + "tag": ["ceo", "ebitda"], } ), ], diff --git a/openbb_platform/openbb/package/equity_fundamental.py b/openbb_platform/openbb/package/equity_fundamental.py index 6716ccbbdcd..c7447aae0bf 100644 --- a/openbb_platform/openbb/package/equity_fundamental.py +++ b/openbb_platform/openbb/package/equity_fundamental.py @@ -1273,7 +1273,7 @@ class ROUTER_equity_fundamental(Container): Optional[int], OpenBBCustomParameter(description="The number of data entries to return."), ] = 1000, - type: Annotated[ + tag_type: Annotated[ Optional[str], OpenBBCustomParameter(description="Filter by type, when applicable."), ] = None, @@ -1300,7 +1300,7 @@ class ROUTER_equity_fundamental(Container): The frequency of the data. limit : Optional[int] The number of data entries to return. - type : Optional[str] + tag_type : Optional[str] Filter by type, when applicable. sort : Optional[Literal['asc', 'desc']] Sort order. @@ -1327,6 +1327,10 @@ class ROUTER_equity_fundamental(Container): -------------------- date : date The date of the data. + symbol : str + Symbol representing the entity requested in the data. + tag : Optional[str] + Tag name for the fetched data. value : Optional[float] The value of the data. @@ -1349,7 +1353,7 @@ class ROUTER_equity_fundamental(Container): "end_date": end_date, "frequency": frequency, "limit": limit, - "type": type, + "tag_type": tag_type, "sort": sort, }, extra_params=kwargs, @@ -2012,7 +2016,7 @@ class ROUTER_equity_fundamental(Container): Returns ------- OBBject - results : LatestAttributes + results : List[LatestAttributes] Serializable results. provider : Optional[Literal['intrinio']] Provider name. @@ -2025,6 +2029,10 @@ class ROUTER_equity_fundamental(Container): LatestAttributes ---------------- + symbol : str + Symbol representing the entity requested in the data. + tag : Optional[str] + Tag name for the fetched data. value : Optional[Union[str, float]] The value of the data. diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py index 4bbaff5af6e..890cb66be2b 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py @@ -1,16 +1,22 @@ """Intrinio Historical Attributes Model.""" +import warnings from datetime import datetime from typing import Any, Dict, List, Optional from dateutil.relativedelta import relativedelta +from openbb_core.app.model.abstract.warning import OpenBBWarning from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.standard_models.historical_attributes import ( HistoricalAttributesData, HistoricalAttributesQueryParams, ) -from openbb_core.provider.utils.helpers import get_querystring -from openbb_intrinio.utils.helpers import get_data_many +from openbb_core.provider.utils.helpers import ( + ClientResponse, + ClientSession, + amake_requests, + get_querystring, +) class IntrinioHistoricalAttributesQueryParams(HistoricalAttributesQueryParams): @@ -19,7 +25,7 @@ class IntrinioHistoricalAttributesQueryParams(HistoricalAttributesQueryParams): Source: https://docs.intrinio.com/documentation/web_api/get_historical_data_v2 """ - __alias_dict__ = {"sort": "sort_order", "limit": "page_size"} + __alias_dict__ = {"sort": "sort_order", "limit": "page_size", "tag_type": "type"} class IntrinioHistoricalAttributesData(HistoricalAttributesData): @@ -62,8 +68,59 @@ class IntrinioHistoricalAttributesFetcher( base_url = "https://api-v2.intrinio.com" query_str = get_querystring(query.model_dump(by_alias=True), ["symbol", "tag"]) - url = f"{base_url}/historical_data/{query.symbol}/{query.tag}?{query_str}&api_key={api_key}" - return await get_data_many(url, "historical_data") + def generate_url(symbol: str, tag: str) -> str: + """Returns the url for the given symbol and tag.""" + url_params = f"{symbol}/{tag}?{query_str}&api_key={api_key}" + url = f"{base_url}/historical_data/{url_params}" + return url + + async def callback( + response: ClientResponse, session: ClientSession + ) -> List[Dict]: + """Return the response.""" + init_response = await response.json() + + if message := init_response.get("error") or init_response.get("message"): + warnings.warn(message=message, category=OpenBBWarning) + return [] + + symbol = response.url.parts[-2] + tag = response.url.parts[-1] + + all_data: list = init_response.get("historical_data", []) + all_data = [{**item, "symbol": symbol, "tag": tag} for item in all_data] + + next_page = init_response.get("next_page", None) + while next_page: + url = response.url.update_query(next_page=next_page).human_repr() + response_data = await session.get_json(url) + + if message := response_data.get("error") or response_data.get( + "message" + ): + warnings.warn(message=message, category=OpenBBWarning) + return [] + + symbol = response.url.parts[-2] + tag = response_data.url.parts[-1] + + response_data = response_data.get("historical_data", []) + response_data = [ + {**item, "symbol": symbol, "tag": tag} for item in response_data + ] + + all_data.extend(response_data) + next_page = response_data.get("next_page", None) + + return all_data + + urls = [ + generate_url(symbol, tag) + for symbol in query.symbol.split(",") + for tag in query.tag.split(",") + ] + + return await amake_requests(urls, callback, **kwargs) @staticmethod def transform_data( diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py index 47cf5eda3c8..0ca153f6123 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py @@ -1,13 +1,18 @@ """Intrinio Latest Attributes Model.""" -from typing import Any, Dict, Optional +import warnings +from typing import Any, Dict, List, Optional +from openbb_core.app.model.abstract.warning import OpenBBWarning from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.standard_models.latest_attributes import ( LatestAttributesData, LatestAttributesQueryParams, ) -from openbb_intrinio.utils.helpers import get_data +from openbb_core.provider.utils.helpers import ( + ClientResponse, + amake_requests, +) class IntrinioLatestAttributesQueryParams(LatestAttributesQueryParams): @@ -25,7 +30,7 @@ class IntrinioLatestAttributesData(LatestAttributesData): class IntrinioLatestAttributesFetcher( Fetcher[ IntrinioLatestAttributesQueryParams, - IntrinioLatestAttributesData, + List[IntrinioLatestAttributesData], ] ): """Transform the query, extract and transform the data from the Intrinio endpoints.""" @@ -43,14 +48,46 @@ class IntrinioLatestAttributesFetcher( ) -> Dict: """Return the raw data from the Intrinio endpoint.""" api_key = credentials.get("intrinio_api_key") if credentials else "" - url = f"https://api-v2.intrinio.com/companies/{query.symbol}/data_point/{query.tag}?api_key={api_key}" - return await get_data(url) + + base_url = "https://api-v2.intrinio.com/companies" + + def generate_url(symbol: str, tag: str) -> str: + """Returns the url for the given symbol and tag.""" + return f"{base_url}/{symbol}/data_point/{tag}?api_key={api_key}" + + async def callback(response: ClientResponse, _: Any) -> Dict: + """Return the response.""" + response_data = await response.json() + + if isinstance(response_data, Dict) and ( + "error" in response_data or "message" in response_data + ): + warnings.warn( + message=response_data.get("error") or response_data.get("message"), + category=OpenBBWarning, + ) + return {} + if not response_data: + return {} + + tag = response.url.parts[-1] + symbol = response.url.parts[-3] + + return {"symbol": symbol, "tag": tag, "value": response_data} + + urls = [ + generate_url(symbol, tag) + for symbol in query.symbol.split(",") + for tag in query.tag.split(",") + ] + + return await amake_requests(urls, callback, **kwargs) @staticmethod def transform_data( query: IntrinioLatestAttributesQueryParams, # pylint: disable=unused-argument data: Dict, **kwargs: Any, - ) -> IntrinioLatestAttributesData: + ) -> List[IntrinioLatestAttributesData]: """Return the transformed data.""" - return IntrinioLatestAttributesData.model_validate(data) + return [IntrinioLatestAttributesData.model_validate(d) for d in data] diff --git a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py index 866e2537150..bc2986f1285 100644 --- a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py +++ b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py @@ -192,11 +192,11 @@ def test_intrinio_search_attributes(credentials=test_credentials): def test_intrinio_historical_attributes(credentials=test_credentials): params = { "provider": "intrinio", - "symbol": "AAPL", - "tag": "ebit", + "symbol": "AAPL,MSFT", + "tag": "ebit,marketcap", "frequency": "yearly", "limit": 1000, - "type": None, + "tag_type": None, "start_date": date(2013, 1, 1), "end_date": date(2023, 1, 1), "sort": "desc", @@ -211,8 +211,8 @@ def test_intrinio_historical_attributes(credentials=test_credentials): def test_intrinio_latest_attributes(credentials=test_credentials): params = { "provider": "intrinio", - "symbol": "AAPL", - "tag": "ceo", + "symbol": "AAPL,MSFT", + "tag": "ceo,marketcap", } fetcher = IntrinioLatestAttributesFetcher() |