diff options
author | Danglewood <85772166+deeleeramone@users.noreply.github.com> | 2024-03-16 14:47:04 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-16 22:47:04 +0100 |
commit | 7b0a8e2f15ee8e7d4f4f4fa29ccb7b34c60a2cbe (patch) | |
tree | 5c5e238499777bdd6d0702fef7df1c79f1b64284 | |
parent | 741bb624fe75cbdccccb2d9ae15d18844fc70551 (diff) |
[HotFix] Fix FMP Intraday (#6229)
* fix fmp intraday
* clean up error messages
12 files changed, 300 insertions, 98 deletions
diff --git a/openbb_platform/extensions/crypto/integration/test_crypto_api.py b/openbb_platform/extensions/crypto/integration/test_crypto_api.py index 161bf9960a8..6d38f1681a5 100644 --- a/openbb_platform/extensions/crypto/integration/test_crypto_api.py +++ b/openbb_platform/extensions/crypto/integration/test_crypto_api.py @@ -52,6 +52,15 @@ def test_crypto_search(params, headers): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "BTCUSD,ETHUSD", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/extensions/crypto/integration/test_crypto_python.py b/openbb_platform/extensions/crypto/integration/test_crypto_python.py index 6d3f49e1ae2..b22c29d4dae 100644 --- a/openbb_platform/extensions/crypto/integration/test_crypto_python.py +++ b/openbb_platform/extensions/crypto/integration/test_crypto_python.py @@ -47,6 +47,15 @@ def test_crypto_search(params, obb): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "BTCUSD,ETHUSD", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/extensions/currency/integration/test_currency_api.py b/openbb_platform/extensions/currency/integration/test_currency_api.py index 90449a69e3f..bc80a5d75bc 100644 --- a/openbb_platform/extensions/currency/integration/test_currency_api.py +++ b/openbb_platform/extensions/currency/integration/test_currency_api.py @@ -72,6 +72,15 @@ def test_currency_search(params, headers): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "EURUSD,USDJPY", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/extensions/currency/integration/test_currency_python.py b/openbb_platform/extensions/currency/integration/test_currency_python.py index 0c91b7b19b0..445e4293838 100644 --- a/openbb_platform/extensions/currency/integration/test_currency_python.py +++ b/openbb_platform/extensions/currency/integration/test_currency_python.py @@ -67,6 +67,15 @@ def test_currency_search(params, obb): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "EURUSD,USDJPY", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/extensions/equity/integration/test_equity_api.py b/openbb_platform/extensions/equity/integration/test_equity_api.py index afe8fdeb22d..3d0da660578 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_api.py +++ b/openbb_platform/extensions/equity/integration/test_equity_api.py @@ -941,6 +941,15 @@ def test_equity_compare_groups(params, headers): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "AAPL,MSFT", + "start_date": None, + "end_date": None, + } + ), + ( + { "timezone": "UTC", "source": "realtime", "start_time": None, diff --git a/openbb_platform/extensions/equity/integration/test_equity_python.py b/openbb_platform/extensions/equity/integration/test_equity_python.py index 71400e8409a..04d79836812 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_python.py +++ b/openbb_platform/extensions/equity/integration/test_equity_python.py @@ -880,6 +880,15 @@ def test_equity_compare_groups(params, obb): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "AAPL,MSFT", + "start_date": None, + "end_date": None, + } + ), + ( + { "timezone": "UTC", "source": "realtime", "start_time": None, diff --git a/openbb_platform/extensions/index/integration/test_index_api.py b/openbb_platform/extensions/index/integration/test_index_api.py index cd6e2868384..0b5006c80f5 100644 --- a/openbb_platform/extensions/index/integration/test_index_api.py +++ b/openbb_platform/extensions/index/integration/test_index_api.py @@ -62,6 +62,15 @@ def test_index_constituents(params, headers): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "^DJI,^NDX", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/extensions/index/integration/test_index_python.py b/openbb_platform/extensions/index/integration/test_index_python.py index 29f20b5f34a..1082fce00ba 100644 --- a/openbb_platform/extensions/index/integration/test_index_python.py +++ b/openbb_platform/extensions/index/integration/test_index_python.py @@ -58,6 +58,15 @@ def test_index_constituents(params, obb): ), ( { + "interval": "1h", + "provider": "fmp", + "symbol": "^DJI,^NDX", + "start_date": None, + "end_date": None, + } + ), + ( + { "interval": "1m", "sort": "desc", "limit": 49999, diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py index ab4c7074f6c..8486d46f69a 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py @@ -2,9 +2,10 @@ # pylint: disable=unused-argument -import warnings +import asyncio from datetime import datetime from typing import Any, Dict, List, Literal, Optional +from warnings import warn from dateutil.relativedelta import relativedelta from openbb_core.provider.abstract.fetcher import Fetcher @@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, QUERY_DESCRIPTIONS, ) +from openbb_core.provider.utils.errors import EmptyDataError from openbb_core.provider.utils.helpers import ( - ClientResponse, - amake_requests, + amake_request, get_querystring, ) from openbb_fmp.utils.helpers import get_interval from pydantic import Field -_warn = warnings.warn - class FMPCryptoHistoricalQueryParams(CryptoHistoricalQueryParams): """ @@ -104,29 +103,55 @@ class FMPCryptoHistoricalFetcher( url = f"{base_url}/historical-price-full/crypto/{url_params}" return url - # if there are more than 20 symbols, we need to increase the timeout - if len(query.symbol.split(",")) > 20: - kwargs.update({"preferences": {"request_timeout": 30}}) + symbols = query.symbol.split(",") + + results = [] + messages = [] + + async def get_one(symbol): + """Get data for one symbol.""" + + url = get_url_params(symbol) + + data = [] + + response = await amake_request(url, **kwargs) - async def callback(response: ClientResponse, _: Any) -> List[Dict]: - data = await response.json() - symbol = response.url.parts[-1] - results = [] - if not data: - _warn(f"No data found the the symbol: {symbol}") - return results + if isinstance(response, dict) and response.get("Error Message"): + message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}" + warn(message) + messages.append(message) - if isinstance(data, dict): - results = data.get("historical", []) + if not response: + message = f"No data found for {symbol}." + warn(message) + messages.append(message) - if "," in query.symbol: - for d in results: - d["symbol"] = symbol - return results + if isinstance(response, list) and len(response) > 0: + data = response + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - urls = [get_url_params(symbol) for symbol in query.symbol.split(",")] + if isinstance(response, dict) and response.get("historical"): + data = response["historical"] + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - return await amake_requests(urls, callback, **kwargs) + if data: + results.extend(data) + + tasks = [get_one(symbol) for symbol in symbols] + + await asyncio.gather(*tasks) + + if not results: + raise EmptyDataError( + f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}" + ) + + return results @staticmethod def transform_data( @@ -138,7 +163,15 @@ class FMPCryptoHistoricalFetcher( to_pop = ["label", "changePercent", "unadjustedVolume"] results: List[FMPCryptoHistoricalData] = [] - for d in sorted(data, key=lambda x: x["date"], reverse=False): + for d in sorted( + data, + key=lambda x: ( + (x["date"], x["symbol"]) + if len(query.symbol.split(",")) > 1 + else x["date"] + ), + reverse=False, + ): _ = [d.pop(pop) for pop in to_pop if pop in d] if d.get("unadjusted_volume") == d.get("volume"): _ = d.pop("unadjusted_volume") diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py index 3c9a9668f82..02fceedc5ee 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py @@ -2,9 +2,10 @@ # pylint: disable=unused-argument -import warnings +import asyncio from datetime import datetime from typing import Any, Dict, List, Literal, Optional +from warnings import warn from dateutil.relativedelta import relativedelta from openbb_core.provider.abstract.fetcher import Fetcher @@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, QUERY_DESCRIPTIONS, ) +from openbb_core.provider.utils.errors import EmptyDataError from openbb_core.provider.utils.helpers import ( - ClientResponse, - amake_requests, + amake_request, get_querystring, ) from openbb_fmp.utils.helpers import get_interval from pydantic import Field -_warn = warnings.warn - class FMPCurrencyHistoricalQueryParams(CurrencyHistoricalQueryParams): """FMP Currency Historical Price Query. @@ -102,29 +101,55 @@ class FMPCurrencyHistoricalFetcher( url = f"{base_url}/historical-price-full/forex/{url_params}" return url - # if there are more than 20 symbols, we need to increase the timeout - if len(query.symbol.split(",")) > 20: - kwargs.update({"preferences": {"request_timeout": 30}}) + symbols = query.symbol.split(",") + + results = [] + messages = [] + + async def get_one(symbol): + """Get data for one symbol.""" + + url = get_url_params(symbol) + + data = [] + + response = await amake_request(url, **kwargs) - async def callback(response: ClientResponse, _: Any) -> List[Dict]: - data = await response.json() - symbol = response.url.parts[-1] - results = [] - if not data: - _warn(f"No data found the the symbol: {symbol}") - return results + if isinstance(response, dict) and response.get("Error Message"): + message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}" + warn(message) + messages.append(message) - if isinstance(data, dict): - results = data.get("historical", []) + if not response: + message = f"No data found for {symbol}." + warn(message) + messages.append(message) - if "," in query.symbol: - for d in results: - d["symbol"] = symbol - return results + if isinstance(response, list) and len(response) > 0: + data = response + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - urls = [get_url_params(symbol) for symbol in query.symbol.split(",")] + if isinstance(response, dict) and response.get("historical"): + data = response["historical"] + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - return await amake_requests(urls, callback, **kwargs) + if data: + results.extend(data) + + tasks = [get_one(symbol) for symbol in symbols] + + await asyncio.gather(*tasks) + + if not results: + raise EmptyDataError( + f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}" + ) + + return results @staticmethod def transform_data( @@ -136,7 +161,15 @@ class FMPCurrencyHistoricalFetcher( to_pop = ["label", "changePercent", "unadjustedVolume"] results: List[FMPCurrencyHistoricalData] = [] - for d in sorted(data, key=lambda x: x["date"], reverse=False): + for d in sorted( + data, + key=lambda x: ( + (x["date"], x["symbol"]) + if len(query.symbol.split(",")) > 1 + else x["date"] + ), + reverse=False, + ): _ = [d.pop(pop) for pop in to_pop if pop in d] if d.get("unadjusted_volume") == d.get("volume"): _ = d.pop("unadjusted_volume") diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py index 79651c71c2f..5225a0daa18 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py @@ -2,9 +2,10 @@ # pylint: disable=unused-argument -import warnings +import asyncio from datetime import datetime from typing import Any, Dict, List, Literal, Optional +from warnings import warn from dateutil.relativedelta import relativedelta from openbb_core.provider.abstract.fetcher import Fetcher @@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, QUERY_DESCRIPTIONS, ) +from openbb_core.provider.utils.errors import EmptyDataError from openbb_core.provider.utils.helpers import ( - ClientResponse, - amake_requests, + amake_request, get_querystring, ) from openbb_fmp.utils.helpers import get_interval from pydantic import Field -_warn = warnings.warn - class FMPEquityHistoricalQueryParams(EquityHistoricalQueryParams): """FMP Equity Historical Price Query. @@ -105,29 +104,55 @@ class FMPEquityHistoricalFetcher( url = f"{base_url}/historical-price-full/{url_params}" return url - # if there are more than 20 symbols, we need to increase the timeout - if len(query.symbol.split(",")) > 20: - kwargs.update({"preferences": {"request_timeout": 30}}) + symbols = query.symbol.split(",") + + results = [] + messages = [] + + async def get_one(symbol): + """Get data for one symbol.""" + + url = get_url_params(symbol) + + data = [] + + response = await amake_request(url, **kwargs) - async def callback(response: ClientResponse, _: Any) -> List[Dict]: - data = await response.json() - symbol = response.url.parts[-1] - results = [] - if not data: - _warn(f"No data found the the symbol: {symbol}") - return results + if isinstance(response, dict) and response.get("Error Message"): + message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}" + warn(message) + messages.append(message) - if isinstance(data, dict): - results = data.get("historical", []) + if not response: + message = f"No data found for {symbol}." + warn(message) + messages.append(message) - if "," in query.symbol: - for d in results: - d["symbol"] = symbol - return results + if isinstance(response, list) and len(response) > 0: + data = response + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - urls = [get_url_params(symbol) for symbol in query.symbol.split(",")] + if isinstance(response, dict) and response.get("historical"): + data = response["historical"] + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - return await amake_requests(urls, callback, **kwargs) + if data: + results.extend(data) + + tasks = [get_one(symbol) for symbol in symbols] + + await asyncio.gather(*tasks) + + if not results: + raise EmptyDataError( + f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}" + ) + + return results @staticmethod def transform_data( @@ -139,7 +164,15 @@ class FMPEquityHistoricalFetcher( to_pop = ["label", "changePercent"] results: List[FMPEquityHistoricalData] = [] - for d in sorted(data, key=lambda x: x["date"], reverse=False): + for d in sorted( + data, + key=lambda x: ( + (x["date"], x["symbol"]) + if len(query.symbol.split(",")) > 1 + else x["date"] + ), + reverse=False, + ): _ = [d.pop(pop) for pop in to_pop if pop in d] if d.get("unadjusted_volume") == d.get("volume"): _ = d.pop("unadjusted_volume") diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py index 13d5582a3f9..33eabd6f5b3 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py @@ -2,9 +2,10 @@ # pylint: disable=unused-argument -import warnings +import asyncio from datetime import datetime from typing import Any, Dict, List, Literal, Optional +from warnings import warn from dateutil.relativedelta import relativedelta from openbb_core.provider.abstract.fetcher import Fetcher @@ -18,15 +19,12 @@ from openbb_core.provider.utils.descriptions import ( ) from openbb_core.provider.utils.errors import EmptyDataError from openbb_core.provider.utils.helpers import ( - ClientResponse, - amake_requests, + amake_request, get_querystring, ) from openbb_fmp.utils.helpers import get_interval from pydantic import Field -_warn = warnings.warn - class FMPIndexHistoricalQueryParams(IndexHistoricalQueryParams): """FMP Index Historical Query. @@ -103,30 +101,55 @@ class FMPIndexHistoricalFetcher( url = f"{base_url}/historical-price-full/{url_params}" return url - # if there are more than 20 symbols, we need to increase the timeout - if len(query.symbol.split(",")) > 20: - kwargs.update({"preferences": {"request_timeout": 30}}) + symbols = query.symbol.split(",") + + results = [] + messages = [] + + async def get_one(symbol): + """Get data for one symbol.""" + + url = get_url_params(symbol) + + data = [] + + response = await amake_request(url, **kwargs) - async def callback(response: ClientResponse, _: Any) -> List[Dict]: - data = await response.json() - symbol = response.url.parts[-1] - results = [] - if not data: - _warn(f"No data found the the symbol: {symbol}") - return results + if isinstance(response, dict) and response.get("Error Message"): + message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}" + warn(message) + messages.append(message) - if isinstance(data, dict): - results = data.get("historical", []) - if isinstance(data, list): - results = data + if not response: + message = f"No data found for {symbol}." + warn(message) + messages.append(message) - if "," in query.symbol: - for d in results: - d["symbol"] = symbol - return results + if isinstance(response, list) and len(response) > 0: + data = response + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol - urls = [get_url_params(symbol) for symbol in query.symbol.split(",")] - return await amake_requests(urls, callback, **kwargs) + if isinstance(response, dict) and response.get("historical"): + data = response["historical"] + if len(symbols) > 1: + for d in data: + d["symbol"] = symbol + + if data: + results.extend(data) + + tasks = [get_one(symbol) for symbol in symbols] + + await asyncio.gather(*tasks) + + if not results: + raise EmptyDataError( + f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}" + ) + + return results @staticmethod def transform_data( @@ -142,7 +165,15 @@ class FMPIndexHistoricalFetcher( to_pop = ["label", "changePercent", "unadjustedVolume", "adjClose"] results: List[FMPIndexHistoricalData] = [] - for d in sorted(data, key=lambda x: x["date"], reverse=False): + for d in sorted( + data, + key=lambda x: ( + (x["date"], x["symbol"]) + if len(query.symbol.split(",")) > 1 + else x["date"] + ), + reverse=False, + ): _ = [d.pop(pop) for pop in to_pop if pop in d] if d.get("volume") == 0: _ = d.pop("volume") |