summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPratyush Shukla <ps4534@nyu.edu>2024-02-02 16:42:50 +0530
committerGitHub <noreply@github.com>2024-02-02 11:12:50 +0000
commit9ddbf34456b748fd38aa7e1bd4e95618dc253f94 (patch)
tree075d343527ab81b9c65156b87b464cd6d2503a41
parent5f8fca5a4a8896a87d5e7f5244627a21f53e3df9 (diff)
[Feature] Add support for multiple tags in `historical_attributes` and `latest_attributes` (#6013)
* historical attributes standard model - * add support multiple tags * add `tag` field in data * add support for multiple tags in historical attributes * set correct return type in `callback` function * latest attributes standard model - * add support for multiple tags * add `tag` field in data * add support for multiple tags in latest attributes * add test params for the attributes endpoints * add support for multiple `symbol` and add `symbol` field in the data * modified code to fetch data from multiple symbols and throw warning as per @deeleeramone * add params for testing multiple symbols * linting * fix test params * fixed condition to check if its a `Dict` type * updated intrinio fetcher tests for the statements * updated static
-rw-r--r--openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py26
-rw-r--r--openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py29
-rw-r--r--openbb_platform/extensions/equity/integration/test_equity_api.py84
-rw-r--r--openbb_platform/extensions/equity/integration/test_equity_python.py82
-rw-r--r--openbb_platform/openbb/package/equity_fundamental.py16
-rw-r--r--openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py67
-rw-r--r--openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py51
-rw-r--r--openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py10
8 files changed, 332 insertions, 33 deletions
diff --git a/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py b/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py
index 8bda172520d..54fa6bda03d 100644
--- a/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py
+++ b/openbb_platform/core/openbb_core/provider/standard_models/historical_attributes.py
@@ -1,9 +1,9 @@
"""Historical Attributes Standard Model."""
from datetime import date as dateType
-from typing import Literal, Optional
+from typing import List, Literal, Optional, Set, Union
-from pydantic import Field
+from pydantic import Field, field_validator
from openbb_core.provider.abstract.data import Data
from openbb_core.provider.abstract.query_params import QueryParams
@@ -30,16 +30,36 @@ class HistoricalAttributesQueryParams(QueryParams):
limit: Optional[int] = Field(
default=1000, description=QUERY_DESCRIPTIONS.get("limit")
)
- type: Optional[str] = Field(
+ tag_type: Optional[str] = Field(
default=None, description="Filter by type, when applicable."
)
sort: Optional[Literal["asc", "desc"]] = Field(
default="desc", description="Sort order."
)
+ @field_validator("tag", mode="before", check_fields=False)
+ @classmethod
+ def multiple_tags(cls, v: Union[str, List[str], Set[str]]):
+ """Accept a comma-separated string or list of tags."""
+ if isinstance(v, str):
+ return v.lower()
+ return ",".join([tag.lower() for tag in list(v)])
+
+ @field_validator("symbol", mode="before", check_fields=False)
+ @classmethod
+ def upper_symbol(cls, v: Union[str, List[str], Set[str]]):
+ """Convert symbol to uppercase."""
+ if isinstance(v, str):
+ return v.upper()
+ return ",".join([symbol.upper() for symbol in list(v)])
+
class HistoricalAttributesData(Data):
"""Historical Attributes Data."""
date: dateType = Field(description=DATA_DESCRIPTIONS.get("date"))
+ symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol"))
+ tag: Optional[str] = Field(
+ default=None, description="Tag name for the fetched data."
+ )
value: Optional[float] = Field(default=None, description="The value of the data.")
diff --git a/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py b/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py
index d4245aa7a7a..e0a8744e516 100644
--- a/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py
+++ b/openbb_platform/core/openbb_core/provider/standard_models/latest_attributes.py
@@ -1,12 +1,15 @@
"""Latest Attributes Standard Model."""
-from typing import Optional, Union
+from typing import List, Optional, Set, Union
-from pydantic import Field
+from pydantic import Field, field_validator
from openbb_core.provider.abstract.data import Data
from openbb_core.provider.abstract.query_params import QueryParams
-from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS
+from openbb_core.provider.utils.descriptions import (
+ DATA_DESCRIPTIONS,
+ QUERY_DESCRIPTIONS,
+)
class LatestAttributesQueryParams(QueryParams):
@@ -15,10 +18,30 @@ class LatestAttributesQueryParams(QueryParams):
symbol: str = Field(description=QUERY_DESCRIPTIONS.get("symbol"))
tag: str = Field(description="Intrinio data tag ID or code.")
+ @field_validator("tag", mode="before", check_fields=False)
+ @classmethod
+ def multiple_tags(cls, v: Union[str, List[str], Set[str]]):
+ """Accept a comma-separated string or list of tags."""
+ if isinstance(v, str):
+ return v.lower()
+ return ",".join([tag.lower() for tag in list(v)])
+
+ @field_validator("symbol", mode="before", check_fields=False)
+ @classmethod
+ def upper_symbol(cls, v: Union[str, List[str], Set[str]]):
+ """Convert symbol to uppercase."""
+ if isinstance(v, str):
+ return v.upper()
+ return ",".join([symbol.upper() for symbol in list(v)])
+
class LatestAttributesData(Data):
"""Latest Attributes Data."""
+ symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol"))
+ tag: Optional[str] = Field(
+ default=None, description="Tag name for the fetched data."
+ )
value: Optional[Union[str, float]] = Field(
default=None, description="The value of the data."
)
diff --git a/openbb_platform/extensions/equity/integration/test_equity_api.py b/openbb_platform/extensions/equity/integration/test_equity_api.py
index ca4b536fc08..6cd0709dcb1 100644
--- a/openbb_platform/extensions/equity/integration/test_equity_api.py
+++ b/openbb_platform/extensions/equity/integration/test_equity_api.py
@@ -1055,7 +1055,59 @@ def test_equity_fundamental_search_attributes(params, headers):
"tag": "ebit",
"frequency": "yearly",
"limit": 1000,
- "type": None,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": "ebit,ebitda,marketcap",
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": ["ebit", "ebitda", "marketcap"],
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL,MSFT",
+ "tag": "ebit,ebitda,marketcap",
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": ["AAPL", "MSFT"],
+ "tag": ["ebit", "ebitda", "marketcap"],
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
"start_date": "2013-01-01",
"end_date": "2023-01-01",
"sort": "desc",
@@ -1087,10 +1139,38 @@ def test_equity_fundamental_historical_attributes(params, headers):
(
{
"provider": "intrinio",
- "symbol": "MSFT",
+ "symbol": "AAPL",
"tag": "ebitda",
}
),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": "ceo,ebitda",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": ["ceo", "ebitda"],
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL,MSFT",
+ "tag": ["ceo", "ebitda"],
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": ["AAPL", "MSFT"],
+ "tag": ["ceo", "ebitda"],
+ }
+ ),
],
)
@pytest.mark.integration
diff --git a/openbb_platform/extensions/equity/integration/test_equity_python.py b/openbb_platform/extensions/equity/integration/test_equity_python.py
index 914f18f9773..09adb9bb956 100644
--- a/openbb_platform/extensions/equity/integration/test_equity_python.py
+++ b/openbb_platform/extensions/equity/integration/test_equity_python.py
@@ -999,7 +999,59 @@ def test_equity_fundamental_search_attributes(params, obb):
"tag": "ebit",
"frequency": "yearly",
"limit": 1000,
- "type": None,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": "ebit,ebitda,marketcap",
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": ["ebit", "ebitda", "marketcap"],
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL,MSFT",
+ "tag": "ebit,ebitda,marketcap",
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
+ "start_date": "2013-01-01",
+ "end_date": "2023-01-01",
+ "sort": "desc",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": ["AAPL", "MSFT"],
+ "tag": ["ebit", "ebitda", "marketcap"],
+ "frequency": "yearly",
+ "limit": 1000,
+ "tag_type": None,
"start_date": "2013-01-01",
"end_date": "2023-01-01",
"sort": "desc",
@@ -1020,6 +1072,7 @@ def test_equity_fundamental_historical_attributes(params, obb):
[
(
{
+ "provider": "intrinio",
"symbol": "AAPL",
"tag": "ceo",
}
@@ -1028,14 +1081,35 @@ def test_equity_fundamental_historical_attributes(params, obb):
{
"provider": "intrinio",
"symbol": "AAPL",
- "tag": "ceo",
+ "tag": "ebitda",
}
),
(
{
"provider": "intrinio",
- "symbol": "MSFT",
- "tag": "ebitda",
+ "symbol": "AAPL",
+ "tag": "ceo,ebitda",
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL",
+ "tag": ["ceo", "ebitda"],
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": "AAPL,MSFT",
+ "tag": ["ceo", "ebitda"],
+ }
+ ),
+ (
+ {
+ "provider": "intrinio",
+ "symbol": ["AAPL", "MSFT"],
+ "tag": ["ceo", "ebitda"],
}
),
],
diff --git a/openbb_platform/openbb/package/equity_fundamental.py b/openbb_platform/openbb/package/equity_fundamental.py
index 6716ccbbdcd..c7447aae0bf 100644
--- a/openbb_platform/openbb/package/equity_fundamental.py
+++ b/openbb_platform/openbb/package/equity_fundamental.py
@@ -1273,7 +1273,7 @@ class ROUTER_equity_fundamental(Container):
Optional[int],
OpenBBCustomParameter(description="The number of data entries to return."),
] = 1000,
- type: Annotated[
+ tag_type: Annotated[
Optional[str],
OpenBBCustomParameter(description="Filter by type, when applicable."),
] = None,
@@ -1300,7 +1300,7 @@ class ROUTER_equity_fundamental(Container):
The frequency of the data.
limit : Optional[int]
The number of data entries to return.
- type : Optional[str]
+ tag_type : Optional[str]
Filter by type, when applicable.
sort : Optional[Literal['asc', 'desc']]
Sort order.
@@ -1327,6 +1327,10 @@ class ROUTER_equity_fundamental(Container):
--------------------
date : date
The date of the data.
+ symbol : str
+ Symbol representing the entity requested in the data.
+ tag : Optional[str]
+ Tag name for the fetched data.
value : Optional[float]
The value of the data.
@@ -1349,7 +1353,7 @@ class ROUTER_equity_fundamental(Container):
"end_date": end_date,
"frequency": frequency,
"limit": limit,
- "type": type,
+ "tag_type": tag_type,
"sort": sort,
},
extra_params=kwargs,
@@ -2012,7 +2016,7 @@ class ROUTER_equity_fundamental(Container):
Returns
-------
OBBject
- results : LatestAttributes
+ results : List[LatestAttributes]
Serializable results.
provider : Optional[Literal['intrinio']]
Provider name.
@@ -2025,6 +2029,10 @@ class ROUTER_equity_fundamental(Container):
LatestAttributes
----------------
+ symbol : str
+ Symbol representing the entity requested in the data.
+ tag : Optional[str]
+ Tag name for the fetched data.
value : Optional[Union[str, float]]
The value of the data.
diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py
index 4bbaff5af6e..890cb66be2b 100644
--- a/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py
+++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/historical_attributes.py
@@ -1,16 +1,22 @@
"""Intrinio Historical Attributes Model."""
+import warnings
from datetime import datetime
from typing import Any, Dict, List, Optional
from dateutil.relativedelta import relativedelta
+from openbb_core.app.model.abstract.warning import OpenBBWarning
from openbb_core.provider.abstract.fetcher import Fetcher
from openbb_core.provider.standard_models.historical_attributes import (
HistoricalAttributesData,
HistoricalAttributesQueryParams,
)
-from openbb_core.provider.utils.helpers import get_querystring
-from openbb_intrinio.utils.helpers import get_data_many
+from openbb_core.provider.utils.helpers import (
+ ClientResponse,
+ ClientSession,
+ amake_requests,
+ get_querystring,
+)
class IntrinioHistoricalAttributesQueryParams(HistoricalAttributesQueryParams):
@@ -19,7 +25,7 @@ class IntrinioHistoricalAttributesQueryParams(HistoricalAttributesQueryParams):
Source: https://docs.intrinio.com/documentation/web_api/get_historical_data_v2
"""
- __alias_dict__ = {"sort": "sort_order", "limit": "page_size"}
+ __alias_dict__ = {"sort": "sort_order", "limit": "page_size", "tag_type": "type"}
class IntrinioHistoricalAttributesData(HistoricalAttributesData):
@@ -62,8 +68,59 @@ class IntrinioHistoricalAttributesFetcher(
base_url = "https://api-v2.intrinio.com"
query_str = get_querystring(query.model_dump(by_alias=True), ["symbol", "tag"])
- url = f"{base_url}/historical_data/{query.symbol}/{query.tag}?{query_str}&api_key={api_key}"
- return await get_data_many(url, "historical_data")
+ def generate_url(symbol: str, tag: str) -> str:
+ """Returns the url for the given symbol and tag."""
+ url_params = f"{symbol}/{tag}?{query_str}&api_key={api_key}"
+ url = f"{base_url}/historical_data/{url_params}"
+ return url
+
+ async def callback(
+ response: ClientResponse, session: ClientSession
+ ) -> List[Dict]:
+ """Return the response."""
+ init_response = await response.json()
+
+ if message := init_response.get("error") or init_response.get("message"):
+ warnings.warn(message=message, category=OpenBBWarning)
+ return []
+
+ symbol = response.url.parts[-2]
+ tag = response.url.parts[-1]
+
+ all_data: list = init_response.get("historical_data", [])
+ all_data = [{**item, "symbol": symbol, "tag": tag} for item in all_data]
+
+ next_page = init_response.get("next_page", None)
+ while next_page:
+ url = response.url.update_query(next_page=next_page).human_repr()
+ response_data = await session.get_json(url)
+
+ if message := response_data.get("error") or response_data.get(
+ "message"
+ ):
+ warnings.warn(message=message, category=OpenBBWarning)
+ return []
+
+ symbol = response.url.parts[-2]
+ tag = response_data.url.parts[-1]
+
+ response_data = response_data.get("historical_data", [])
+ response_data = [
+ {**item, "symbol": symbol, "tag": tag} for item in response_data
+ ]
+
+ all_data.extend(response_data)
+ next_page = response_data.get("next_page", None)
+
+ return all_data
+
+ urls = [
+ generate_url(symbol, tag)
+ for symbol in query.symbol.split(",")
+ for tag in query.tag.split(",")
+ ]
+
+ return await amake_requests(urls, callback, **kwargs)
@staticmethod
def transform_data(
diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py
index 47cf5eda3c8..0ca153f6123 100644
--- a/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py
+++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/latest_attributes.py
@@ -1,13 +1,18 @@
"""Intrinio Latest Attributes Model."""
-from typing import Any, Dict, Optional
+import warnings
+from typing import Any, Dict, List, Optional
+from openbb_core.app.model.abstract.warning import OpenBBWarning
from openbb_core.provider.abstract.fetcher import Fetcher
from openbb_core.provider.standard_models.latest_attributes import (
LatestAttributesData,
LatestAttributesQueryParams,
)
-from openbb_intrinio.utils.helpers import get_data
+from openbb_core.provider.utils.helpers import (
+ ClientResponse,
+ amake_requests,
+)
class IntrinioLatestAttributesQueryParams(LatestAttributesQueryParams):
@@ -25,7 +30,7 @@ class IntrinioLatestAttributesData(LatestAttributesData):
class IntrinioLatestAttributesFetcher(
Fetcher[
IntrinioLatestAttributesQueryParams,
- IntrinioLatestAttributesData,
+ List[IntrinioLatestAttributesData],
]
):
"""Transform the query, extract and transform the data from the Intrinio endpoints."""
@@ -43,14 +48,46 @@ class IntrinioLatestAttributesFetcher(
) -> Dict:
"""Return the raw data from the Intrinio endpoint."""
api_key = credentials.get("intrinio_api_key") if credentials else ""
- url = f"https://api-v2.intrinio.com/companies/{query.symbol}/data_point/{query.tag}?api_key={api_key}"
- return await get_data(url)
+
+ base_url = "https://api-v2.intrinio.com/companies"
+
+ def generate_url(symbol: str, tag: str) -> str:
+ """Returns the url for the given symbol and tag."""
+ return f"{base_url}/{symbol}/data_point/{tag}?api_key={api_key}"
+
+ async def callback(response: ClientResponse, _: Any) -> Dict:
+ """Return the response."""
+ response_data = await response.json()
+
+ if isinstance(response_data, Dict) and (
+ "error" in response_data or "message" in response_data
+ ):
+ warnings.warn(
+ message=response_data.get("error") or response_data.get("message"),
+ category=OpenBBWarning,
+ )
+ return {}
+ if not response_data:
+ return {}
+
+ tag = response.url.parts[-1]
+ symbol = response.url.parts[-3]
+
+ return {"symbol": symbol, "tag": tag, "value": response_data}
+
+ urls = [
+ generate_url(symbol, tag)
+ for symbol in query.symbol.split(",")
+ for tag in query.tag.split(",")
+ ]
+
+ return await amake_requests(urls, callback, **kwargs)
@staticmethod
def transform_data(
query: IntrinioLatestAttributesQueryParams, # pylint: disable=unused-argument
data: Dict,
**kwargs: Any,
- ) -> IntrinioLatestAttributesData:
+ ) -> List[IntrinioLatestAttributesData]:
"""Return the transformed data."""
- return IntrinioLatestAttributesData.model_validate(data)
+ return [IntrinioLatestAttributesData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py
index 866e2537150..bc2986f1285 100644
--- a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py
+++ b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py
@@ -192,11 +192,11 @@ def test_intrinio_search_attributes(credentials=test_credentials):
def test_intrinio_historical_attributes(credentials=test_credentials):
params = {
"provider": "intrinio",
- "symbol": "AAPL",
- "tag": "ebit",
+ "symbol": "AAPL,MSFT",
+ "tag": "ebit,marketcap",
"frequency": "yearly",
"limit": 1000,
- "type": None,
+ "tag_type": None,
"start_date": date(2013, 1, 1),
"end_date": date(2023, 1, 1),
"sort": "desc",
@@ -211,8 +211,8 @@ def test_intrinio_historical_attributes(credentials=test_credentials):
def test_intrinio_latest_attributes(credentials=test_credentials):
params = {
"provider": "intrinio",
- "symbol": "AAPL",
- "tag": "ceo",
+ "symbol": "AAPL,MSFT",
+ "tag": "ceo,marketcap",
}
fetcher = IntrinioLatestAttributesFetcher()