summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDanglewood <85772166+deeleeramone@users.noreply.github.com>2024-03-16 14:47:04 -0700
committerGitHub <noreply@github.com>2024-03-16 22:47:04 +0100
commit7b0a8e2f15ee8e7d4f4f4fa29ccb7b34c60a2cbe (patch)
tree5c5e238499777bdd6d0702fef7df1c79f1b64284
parent741bb624fe75cbdccccb2d9ae15d18844fc70551 (diff)
[HotFix] Fix FMP Intraday (#6229)
* fix fmp intraday * clean up error messages
-rw-r--r--openbb_platform/extensions/crypto/integration/test_crypto_api.py9
-rw-r--r--openbb_platform/extensions/crypto/integration/test_crypto_python.py9
-rw-r--r--openbb_platform/extensions/currency/integration/test_currency_api.py9
-rw-r--r--openbb_platform/extensions/currency/integration/test_currency_python.py9
-rw-r--r--openbb_platform/extensions/equity/integration/test_equity_api.py9
-rw-r--r--openbb_platform/extensions/equity/integration/test_equity_python.py9
-rw-r--r--openbb_platform/extensions/index/integration/test_index_api.py9
-rw-r--r--openbb_platform/extensions/index/integration/test_index_python.py9
-rw-r--r--openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py81
-rw-r--r--openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py81
-rw-r--r--openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py81
-rw-r--r--openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py83
12 files changed, 300 insertions, 98 deletions
diff --git a/openbb_platform/extensions/crypto/integration/test_crypto_api.py b/openbb_platform/extensions/crypto/integration/test_crypto_api.py
index 161bf9960a8..6d38f1681a5 100644
--- a/openbb_platform/extensions/crypto/integration/test_crypto_api.py
+++ b/openbb_platform/extensions/crypto/integration/test_crypto_api.py
@@ -52,6 +52,15 @@ def test_crypto_search(params, headers):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "BTCUSD,ETHUSD",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/extensions/crypto/integration/test_crypto_python.py b/openbb_platform/extensions/crypto/integration/test_crypto_python.py
index 6d3f49e1ae2..b22c29d4dae 100644
--- a/openbb_platform/extensions/crypto/integration/test_crypto_python.py
+++ b/openbb_platform/extensions/crypto/integration/test_crypto_python.py
@@ -47,6 +47,15 @@ def test_crypto_search(params, obb):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "BTCUSD,ETHUSD",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/extensions/currency/integration/test_currency_api.py b/openbb_platform/extensions/currency/integration/test_currency_api.py
index 90449a69e3f..bc80a5d75bc 100644
--- a/openbb_platform/extensions/currency/integration/test_currency_api.py
+++ b/openbb_platform/extensions/currency/integration/test_currency_api.py
@@ -72,6 +72,15 @@ def test_currency_search(params, headers):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "EURUSD,USDJPY",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/extensions/currency/integration/test_currency_python.py b/openbb_platform/extensions/currency/integration/test_currency_python.py
index 0c91b7b19b0..445e4293838 100644
--- a/openbb_platform/extensions/currency/integration/test_currency_python.py
+++ b/openbb_platform/extensions/currency/integration/test_currency_python.py
@@ -67,6 +67,15 @@ def test_currency_search(params, obb):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "EURUSD,USDJPY",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/extensions/equity/integration/test_equity_api.py b/openbb_platform/extensions/equity/integration/test_equity_api.py
index afe8fdeb22d..3d0da660578 100644
--- a/openbb_platform/extensions/equity/integration/test_equity_api.py
+++ b/openbb_platform/extensions/equity/integration/test_equity_api.py
@@ -941,6 +941,15 @@ def test_equity_compare_groups(params, headers):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "AAPL,MSFT",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"timezone": "UTC",
"source": "realtime",
"start_time": None,
diff --git a/openbb_platform/extensions/equity/integration/test_equity_python.py b/openbb_platform/extensions/equity/integration/test_equity_python.py
index 71400e8409a..04d79836812 100644
--- a/openbb_platform/extensions/equity/integration/test_equity_python.py
+++ b/openbb_platform/extensions/equity/integration/test_equity_python.py
@@ -880,6 +880,15 @@ def test_equity_compare_groups(params, obb):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "AAPL,MSFT",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"timezone": "UTC",
"source": "realtime",
"start_time": None,
diff --git a/openbb_platform/extensions/index/integration/test_index_api.py b/openbb_platform/extensions/index/integration/test_index_api.py
index cd6e2868384..0b5006c80f5 100644
--- a/openbb_platform/extensions/index/integration/test_index_api.py
+++ b/openbb_platform/extensions/index/integration/test_index_api.py
@@ -62,6 +62,15 @@ def test_index_constituents(params, headers):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "^DJI,^NDX",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/extensions/index/integration/test_index_python.py b/openbb_platform/extensions/index/integration/test_index_python.py
index 29f20b5f34a..1082fce00ba 100644
--- a/openbb_platform/extensions/index/integration/test_index_python.py
+++ b/openbb_platform/extensions/index/integration/test_index_python.py
@@ -58,6 +58,15 @@ def test_index_constituents(params, obb):
),
(
{
+ "interval": "1h",
+ "provider": "fmp",
+ "symbol": "^DJI,^NDX",
+ "start_date": None,
+ "end_date": None,
+ }
+ ),
+ (
+ {
"interval": "1m",
"sort": "desc",
"limit": 49999,
diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py
index ab4c7074f6c..8486d46f69a 100644
--- a/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py
+++ b/openbb_platform/providers/fmp/openbb_fmp/models/crypto_historical.py
@@ -2,9 +2,10 @@
# pylint: disable=unused-argument
-import warnings
+import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
+from warnings import warn
from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
@@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import (
DATA_DESCRIPTIONS,
QUERY_DESCRIPTIONS,
)
+from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
- ClientResponse,
- amake_requests,
+ amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field
-_warn = warnings.warn
-
class FMPCryptoHistoricalQueryParams(CryptoHistoricalQueryParams):
"""
@@ -104,29 +103,55 @@ class FMPCryptoHistoricalFetcher(
url = f"{base_url}/historical-price-full/crypto/{url_params}"
return url
- # if there are more than 20 symbols, we need to increase the timeout
- if len(query.symbol.split(",")) > 20:
- kwargs.update({"preferences": {"request_timeout": 30}})
+ symbols = query.symbol.split(",")
+
+ results = []
+ messages = []
+
+ async def get_one(symbol):
+ """Get data for one symbol."""
+
+ url = get_url_params(symbol)
+
+ data = []
+
+ response = await amake_request(url, **kwargs)
- async def callback(response: ClientResponse, _: Any) -> List[Dict]:
- data = await response.json()
- symbol = response.url.parts[-1]
- results = []
- if not data:
- _warn(f"No data found the the symbol: {symbol}")
- return results
+ if isinstance(response, dict) and response.get("Error Message"):
+ message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
+ warn(message)
+ messages.append(message)
- if isinstance(data, dict):
- results = data.get("historical", [])
+ if not response:
+ message = f"No data found for {symbol}."
+ warn(message)
+ messages.append(message)
- if "," in query.symbol:
- for d in results:
- d["symbol"] = symbol
- return results
+ if isinstance(response, list) and len(response) > 0:
+ data = response
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
+ if isinstance(response, dict) and response.get("historical"):
+ data = response["historical"]
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- return await amake_requests(urls, callback, **kwargs)
+ if data:
+ results.extend(data)
+
+ tasks = [get_one(symbol) for symbol in symbols]
+
+ await asyncio.gather(*tasks)
+
+ if not results:
+ raise EmptyDataError(
+ f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
+ )
+
+ return results
@staticmethod
def transform_data(
@@ -138,7 +163,15 @@ class FMPCryptoHistoricalFetcher(
to_pop = ["label", "changePercent", "unadjustedVolume"]
results: List[FMPCryptoHistoricalData] = []
- for d in sorted(data, key=lambda x: x["date"], reverse=False):
+ for d in sorted(
+ data,
+ key=lambda x: (
+ (x["date"], x["symbol"])
+ if len(query.symbol.split(",")) > 1
+ else x["date"]
+ ),
+ reverse=False,
+ ):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("unadjusted_volume") == d.get("volume"):
_ = d.pop("unadjusted_volume")
diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py
index 3c9a9668f82..02fceedc5ee 100644
--- a/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py
+++ b/openbb_platform/providers/fmp/openbb_fmp/models/currency_historical.py
@@ -2,9 +2,10 @@
# pylint: disable=unused-argument
-import warnings
+import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
+from warnings import warn
from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
@@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import (
DATA_DESCRIPTIONS,
QUERY_DESCRIPTIONS,
)
+from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
- ClientResponse,
- amake_requests,
+ amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field
-_warn = warnings.warn
-
class FMPCurrencyHistoricalQueryParams(CurrencyHistoricalQueryParams):
"""FMP Currency Historical Price Query.
@@ -102,29 +101,55 @@ class FMPCurrencyHistoricalFetcher(
url = f"{base_url}/historical-price-full/forex/{url_params}"
return url
- # if there are more than 20 symbols, we need to increase the timeout
- if len(query.symbol.split(",")) > 20:
- kwargs.update({"preferences": {"request_timeout": 30}})
+ symbols = query.symbol.split(",")
+
+ results = []
+ messages = []
+
+ async def get_one(symbol):
+ """Get data for one symbol."""
+
+ url = get_url_params(symbol)
+
+ data = []
+
+ response = await amake_request(url, **kwargs)
- async def callback(response: ClientResponse, _: Any) -> List[Dict]:
- data = await response.json()
- symbol = response.url.parts[-1]
- results = []
- if not data:
- _warn(f"No data found the the symbol: {symbol}")
- return results
+ if isinstance(response, dict) and response.get("Error Message"):
+ message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
+ warn(message)
+ messages.append(message)
- if isinstance(data, dict):
- results = data.get("historical", [])
+ if not response:
+ message = f"No data found for {symbol}."
+ warn(message)
+ messages.append(message)
- if "," in query.symbol:
- for d in results:
- d["symbol"] = symbol
- return results
+ if isinstance(response, list) and len(response) > 0:
+ data = response
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
+ if isinstance(response, dict) and response.get("historical"):
+ data = response["historical"]
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- return await amake_requests(urls, callback, **kwargs)
+ if data:
+ results.extend(data)
+
+ tasks = [get_one(symbol) for symbol in symbols]
+
+ await asyncio.gather(*tasks)
+
+ if not results:
+ raise EmptyDataError(
+ f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
+ )
+
+ return results
@staticmethod
def transform_data(
@@ -136,7 +161,15 @@ class FMPCurrencyHistoricalFetcher(
to_pop = ["label", "changePercent", "unadjustedVolume"]
results: List[FMPCurrencyHistoricalData] = []
- for d in sorted(data, key=lambda x: x["date"], reverse=False):
+ for d in sorted(
+ data,
+ key=lambda x: (
+ (x["date"], x["symbol"])
+ if len(query.symbol.split(",")) > 1
+ else x["date"]
+ ),
+ reverse=False,
+ ):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("unadjusted_volume") == d.get("volume"):
_ = d.pop("unadjusted_volume")
diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py
index 79651c71c2f..5225a0daa18 100644
--- a/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py
+++ b/openbb_platform/providers/fmp/openbb_fmp/models/equity_historical.py
@@ -2,9 +2,10 @@
# pylint: disable=unused-argument
-import warnings
+import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
+from warnings import warn
from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
@@ -16,16 +17,14 @@ from openbb_core.provider.utils.descriptions import (
DATA_DESCRIPTIONS,
QUERY_DESCRIPTIONS,
)
+from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
- ClientResponse,
- amake_requests,
+ amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field
-_warn = warnings.warn
-
class FMPEquityHistoricalQueryParams(EquityHistoricalQueryParams):
"""FMP Equity Historical Price Query.
@@ -105,29 +104,55 @@ class FMPEquityHistoricalFetcher(
url = f"{base_url}/historical-price-full/{url_params}"
return url
- # if there are more than 20 symbols, we need to increase the timeout
- if len(query.symbol.split(",")) > 20:
- kwargs.update({"preferences": {"request_timeout": 30}})
+ symbols = query.symbol.split(",")
+
+ results = []
+ messages = []
+
+ async def get_one(symbol):
+ """Get data for one symbol."""
+
+ url = get_url_params(symbol)
+
+ data = []
+
+ response = await amake_request(url, **kwargs)
- async def callback(response: ClientResponse, _: Any) -> List[Dict]:
- data = await response.json()
- symbol = response.url.parts[-1]
- results = []
- if not data:
- _warn(f"No data found the the symbol: {symbol}")
- return results
+ if isinstance(response, dict) and response.get("Error Message"):
+ message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
+ warn(message)
+ messages.append(message)
- if isinstance(data, dict):
- results = data.get("historical", [])
+ if not response:
+ message = f"No data found for {symbol}."
+ warn(message)
+ messages.append(message)
- if "," in query.symbol:
- for d in results:
- d["symbol"] = symbol
- return results
+ if isinstance(response, list) and len(response) > 0:
+ data = response
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
+ if isinstance(response, dict) and response.get("historical"):
+ data = response["historical"]
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- return await amake_requests(urls, callback, **kwargs)
+ if data:
+ results.extend(data)
+
+ tasks = [get_one(symbol) for symbol in symbols]
+
+ await asyncio.gather(*tasks)
+
+ if not results:
+ raise EmptyDataError(
+ f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
+ )
+
+ return results
@staticmethod
def transform_data(
@@ -139,7 +164,15 @@ class FMPEquityHistoricalFetcher(
to_pop = ["label", "changePercent"]
results: List[FMPEquityHistoricalData] = []
- for d in sorted(data, key=lambda x: x["date"], reverse=False):
+ for d in sorted(
+ data,
+ key=lambda x: (
+ (x["date"], x["symbol"])
+ if len(query.symbol.split(",")) > 1
+ else x["date"]
+ ),
+ reverse=False,
+ ):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("unadjusted_volume") == d.get("volume"):
_ = d.pop("unadjusted_volume")
diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py b/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py
index 13d5582a3f9..33eabd6f5b3 100644
--- a/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py
+++ b/openbb_platform/providers/fmp/openbb_fmp/models/index_historical.py
@@ -2,9 +2,10 @@
# pylint: disable=unused-argument
-import warnings
+import asyncio
from datetime import datetime
from typing import Any, Dict, List, Literal, Optional
+from warnings import warn
from dateutil.relativedelta import relativedelta
from openbb_core.provider.abstract.fetcher import Fetcher
@@ -18,15 +19,12 @@ from openbb_core.provider.utils.descriptions import (
)
from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
- ClientResponse,
- amake_requests,
+ amake_request,
get_querystring,
)
from openbb_fmp.utils.helpers import get_interval
from pydantic import Field
-_warn = warnings.warn
-
class FMPIndexHistoricalQueryParams(IndexHistoricalQueryParams):
"""FMP Index Historical Query.
@@ -103,30 +101,55 @@ class FMPIndexHistoricalFetcher(
url = f"{base_url}/historical-price-full/{url_params}"
return url
- # if there are more than 20 symbols, we need to increase the timeout
- if len(query.symbol.split(",")) > 20:
- kwargs.update({"preferences": {"request_timeout": 30}})
+ symbols = query.symbol.split(",")
+
+ results = []
+ messages = []
+
+ async def get_one(symbol):
+ """Get data for one symbol."""
+
+ url = get_url_params(symbol)
+
+ data = []
+
+ response = await amake_request(url, **kwargs)
- async def callback(response: ClientResponse, _: Any) -> List[Dict]:
- data = await response.json()
- symbol = response.url.parts[-1]
- results = []
- if not data:
- _warn(f"No data found the the symbol: {symbol}")
- return results
+ if isinstance(response, dict) and response.get("Error Message"):
+ message = f"Error fetching data for {symbol}: {response.get('Error Message', '')}"
+ warn(message)
+ messages.append(message)
- if isinstance(data, dict):
- results = data.get("historical", [])
- if isinstance(data, list):
- results = data
+ if not response:
+ message = f"No data found for {symbol}."
+ warn(message)
+ messages.append(message)
- if "," in query.symbol:
- for d in results:
- d["symbol"] = symbol
- return results
+ if isinstance(response, list) and len(response) > 0:
+ data = response
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
- urls = [get_url_params(symbol) for symbol in query.symbol.split(",")]
- return await amake_requests(urls, callback, **kwargs)
+ if isinstance(response, dict) and response.get("historical"):
+ data = response["historical"]
+ if len(symbols) > 1:
+ for d in data:
+ d["symbol"] = symbol
+
+ if data:
+ results.extend(data)
+
+ tasks = [get_one(symbol) for symbol in symbols]
+
+ await asyncio.gather(*tasks)
+
+ if not results:
+ raise EmptyDataError(
+ f"{str(','.join(messages)).replace(',',' ') if messages else 'No data found'}"
+ )
+
+ return results
@staticmethod
def transform_data(
@@ -142,7 +165,15 @@ class FMPIndexHistoricalFetcher(
to_pop = ["label", "changePercent", "unadjustedVolume", "adjClose"]
results: List[FMPIndexHistoricalData] = []
- for d in sorted(data, key=lambda x: x["date"], reverse=False):
+ for d in sorted(
+ data,
+ key=lambda x: (
+ (x["date"], x["symbol"])
+ if len(query.symbol.split(",")) > 1
+ else x["date"]
+ ),
+ reverse=False,
+ ):
_ = [d.pop(pop) for pop in to_pop if pop in d]
if d.get("volume") == 0:
_ = d.pop("volume")