summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJames Maslek <jmaslek11@gmail.com>2024-03-01 13:38:59 -0500
committerGitHub <noreply@github.com>2024-03-01 18:38:59 +0000
commit5f5fad05e75eae92e419f7b9661c2b010ad91d59 (patch)
tree9624d60277646408a05f90b5d916283a0455b811
parentba18734f8e1db035bd6b5185e08be3876fa6e144 (diff)
Improve the OECD requests (#6152)
* remove ultima + althub * move twitter keys * Enhance caching + more specific urls * Update urls for stir * LTIR * cli * one v3 file to fix * Typing edits + gdp dates + gdp all countries * Tests * lint * not sure why those didnt record * handle the fact that we artifically add some start dates and end dates * fix dates with actual solution not weird patch * lint
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/composite_leading_indicator.py19
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/gdp_forecast.py13
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/gdp_nominal.py18
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/gdp_real.py11
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/long_term_interest_rate.py25
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/short_term_interest_rate.py25
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/models/unemployment.py30
-rw-r--r--openbb_platform/providers/oecd/openbb_oecd/utils/helpers.py68
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_forecast_gdp_fetcher.yaml4
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_nominal_gdp_fetcher.yaml4
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_real_gdp_fetcher.yaml4
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_unemployment_fetcher.yaml100434
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecdcli_fetcher.yaml5952
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecdltir_fetcher.yaml8986
-rw-r--r--openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecdstir_fetcher.yaml8700
-rw-r--r--openbb_terminal/reports/templates/economy.ipynb39
16 files changed, 1309 insertions, 123023 deletions
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/composite_leading_indicator.py b/openbb_platform/providers/oecd/openbb_oecd/models/composite_leading_indicator.py
index 8184bea4420..38eab712ebb 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/composite_leading_indicator.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/composite_leading_indicator.py
@@ -82,15 +82,24 @@ class OECDCLIFetcher(Fetcher[OECDCLIQueryParams, List[OECDCLIData]]):
query: OECDCLIQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
- url = "https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/..LI...."
+ country = "" if query.country == "all" else COUNTRY_TO_CODE_CLI[query.country]
+
+ # Note this is only available monthly from OECD
+ url = f"https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/{country}.M.LI...."
+
+ query_dict = {
+ k: v
+ for k, v in query.__dict__.items()
+ if k not in ["start_date", "end_date"]
+ }
data = helpers.get_possibly_cached_data(
- url, function="economy_composite_leading_indicator"
+ url, function="economy_composite_leading_indicator", query_dict=query_dict
)
if query.country != "all":
- data = data.query(f"REF_AREA == '{COUNTRY_TO_CODE_CLI[query.country]}'")
+ data = data.query(f"REF_AREA == '{country}'")
# Filter down
data = data.reset_index(drop=True)[["REF_AREA", "TIME_PERIOD", "VALUE"]].rename(
@@ -109,7 +118,7 @@ class OECDCLIFetcher(Fetcher[OECDCLIQueryParams, List[OECDCLIData]]):
@staticmethod
def transform_data(
query: OECDCLIQueryParams,
- data: Dict,
+ data: List[Dict],
**kwargs: Any,
) -> List[OECDCLIData]:
"""Transform the data from the OECD endpoint."""
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_forecast.py b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_forecast.py
index a204919fdbc..dcf61e6a0b6 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_forecast.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_forecast.py
@@ -15,6 +15,8 @@ from pydantic import Field, field_validator
gdp_countries = tuple(constants.COUNTRY_TO_CODE_GDP_FORECAST.keys())
GDPCountriesLiteral = Literal[gdp_countries] # type: ignore
+# pylint: disable=unused-argument
+
class OECDGdpForecastQueryParams(GdpForecastQueryParams):
"""OECD GDP Forecast Query."""
@@ -67,7 +69,7 @@ class OECDGdpForecastFetcher(
query: OECDGdpForecastQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
units = query.period[0].upper()
_type = "REAL" if query.type == "real" else "NOM"
@@ -94,11 +96,18 @@ class OECDGdpForecastFetcher(
)
data_df = data_df[data_df["country"] == query.country]
data_df = data_df[["country", "date", "value"]]
+ data_df["date"] = data_df["date"].apply(helpers.oecd_date_to_python_date)
+ data_df = data_df[
+ (data_df["date"] <= query.end_date) & (data_df["date"] >= query.start_date)
+ ]
+ data_df["date"] = data_df["date"].apply(
+ lambda x: x.year
+ ) # Validator won't accept datetime.date?
return data_df.to_dict(orient="records")
@staticmethod
def transform_data(
- query: OECDGdpForecastQueryParams, data: Dict, **kwargs: Any
+ query: OECDGdpForecastQueryParams, data: List[Dict], **kwargs: Any
) -> List[OECDGdpForecastData]:
"""Transform the data from the OECD endpoint."""
return [OECDGdpForecastData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_nominal.py b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_nominal.py
index 6c3a3f7ca58..8caea488e24 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_nominal.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_nominal.py
@@ -11,9 +11,11 @@ from openbb_core.provider.standard_models.gdp_nominal import (
from openbb_oecd.utils import constants, helpers
from pydantic import Field, field_validator
-gdp_countries = tuple(constants.COUNTRY_TO_CODE_GDP.keys())
+gdp_countries = tuple(constants.COUNTRY_TO_CODE_GDP.keys()) + ("all",)
GDPCountriesLiteral = Literal[gdp_countries] # type: ignore
+# pylint: disable=unused-argument
+
class OECDGdpNominalQueryParams(GdpNominalQueryParams):
"""OECD Nominal GDP Query."""
@@ -56,7 +58,7 @@ class OECDGdpNominalFetcher(
query: OECDGdpNominalQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
unit = "MLN_USD" if query.units == "usd" else "USD_CAP"
url = (
@@ -76,13 +78,21 @@ class OECDGdpNominalFetcher(
}
)
data_df["country"] = data_df["country"].map(constants.CODE_TO_COUNTRY_GDP)
- data_df = data_df[data_df["country"] == query.country]
+ if query.country != "all":
+ data_df = data_df[data_df["country"] == query.country]
data_df = data_df[["country", "date", "value"]]
+ data_df["date"] = data_df["date"].apply(helpers.oecd_date_to_python_date)
+ data_df = data_df[
+ (data_df["date"] <= query.end_date) & (data_df["date"] >= query.start_date)
+ ]
+ data_df["date"] = data_df["date"].apply(
+ lambda x: x.year
+ ) # Validator won't accept datetime.date?
return data_df.to_dict(orient="records")
@staticmethod
def transform_data(
- query: OECDGdpNominalQueryParams, data: Dict, **kwargs: Any
+ query: OECDGdpNominalQueryParams, data: List[Dict], **kwargs: Any
) -> List[OECDGdpNominalData]:
"""Transform the data from the OECD endpoint."""
return [OECDGdpNominalData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_real.py b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_real.py
index 28021efdbc4..e11a371eda7 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/gdp_real.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/gdp_real.py
@@ -11,10 +11,11 @@ from openbb_core.provider.standard_models.gdp_real import (
from openbb_oecd.utils import constants, helpers
from pydantic import Field, field_validator
-rgdp_countries = tuple(constants.COUNTRY_TO_CODE_RGDP.keys())
+rgdp_countries = tuple(constants.COUNTRY_TO_CODE_RGDP.keys()) + ("all",)
RGDPCountriesLiteral = Literal[rgdp_countries] # type: ignore
+# pylint: disable=unused-argument
class OECDGdpRealQueryParams(GdpRealQueryParams):
"""OECD Real GDP Query."""
@@ -83,8 +84,14 @@ class OECDGdpRealFetcher(Fetcher[OECDGdpRealQueryParams, List[OECDGdpRealData]])
}
)
data_df["country"] = data_df["country"].map(constants.CODE_TO_COUNTRY_RGDP)
- data_df = data_df[data_df["country"] == query.country]
+ if query.country != "all":
+ data_df = data_df[data_df["country"] == query.country]
data_df = data_df[["country", "date", "value"]]
+
+ data_df["date"] = data_df["date"].apply(helpers.oecd_date_to_python_date)
+ data_df = data_df[
+ (data_df["date"] <= query.end_date) & (data_df["date"] >= query.start_date)
+ ]
return data_df.to_dict(orient="records")
@staticmethod
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/long_term_interest_rate.py b/openbb_platform/providers/oecd/openbb_oecd/models/long_term_interest_rate.py
index 67b536bb8c7..9ac5c306eac 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/long_term_interest_rate.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/long_term_interest_rate.py
@@ -87,13 +87,18 @@ class OECDLTIRFetcher(Fetcher[OECDLTIRQueryParams, List[OECDLTIRData]]):
query: OECDLTIRQueryParams, # pylint: disable=W0613
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
frequency = query.frequency[0].upper()
country = "" if query.country == "all" else COUNTRY_TO_CODE_IR[query.country]
- url = "https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/..IRLT...."
+ query_dict = {
+ k: v
+ for k, v in query.__dict__.items()
+ if k not in ["start_date", "end_date"]
+ }
+ url = f"https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/{country}.{frequency}.IRLT...."
data = helpers.get_possibly_cached_data(
- url, function="economy_long_term_interest_rate"
+ url, function="economy_long_term_interest_rate", query_dict=query_dict
)
url_query = f"FREQ=='{frequency}'"
url_query = url_query + f" & REF_AREA=='{country}'" if country else url_query
@@ -107,17 +112,15 @@ class OECDLTIRFetcher(Fetcher[OECDLTIRQueryParams, List[OECDLTIRData]]):
)
data["country"] = data["country"].map(CODE_TO_COUNTRY_IR)
data = data.fillna("N/A").replace("N/A", None)
- data = data.to_dict(orient="records")
-
- start_date = query.start_date.strftime("%Y-%m-%d") # type: ignore
- end_date = query.end_date.strftime("%Y-%m-%d") # type: ignore
- data = list(filter(lambda x: start_date <= x["date"] <= end_date, data))
-
- return data
+ data["date"] = data["date"].apply(helpers.oecd_date_to_python_date)
+ data = data[
+ (data["date"] <= query.end_date) & (data["date"] >= query.start_date)
+ ]
+ return data.to_dict(orient="records")
@staticmethod
def transform_data(
- query: OECDLTIRQueryParams, data: Dict, **kwargs: Any
+ query: OECDLTIRQueryParams, data: List[Dict], **kwargs: Any
) -> List[OECDLTIRData]:
"""Transform the data from the OECD endpoint."""
return [OECDLTIRData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/short_term_interest_rate.py b/openbb_platform/providers/oecd/openbb_oecd/models/short_term_interest_rate.py
index cd018390999..aee3e11aef5 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/short_term_interest_rate.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/short_term_interest_rate.py
@@ -87,13 +87,19 @@ class OECDSTIRFetcher(Fetcher[OECDSTIRQueryParams, List[OECDSTIRData]]):
query: OECDSTIRQueryParams, # pylint: disable=W0613
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
frequency = query.frequency[0].upper()
country = "" if query.country == "all" else COUNTRY_TO_CODE_IR[query.country]
- url = "https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/..IR3TIB...."
+ query_dict = {
+ k: v
+ for k, v in query.__dict__.items()
+ if k not in ["start_date", "end_date"]
+ }
+
+ url = f"https://sdmx.oecd.org/public/rest/data/OECD.SDD.STES,DSD_KEI@DF_KEI,4.0/{country}.{frequency}.IR3TIB...."
data = helpers.get_possibly_cached_data(
- url, function="economy_short_term_interest_rate"
+ url, function="economy_short_term_interest_rate", query_dict=query_dict
)
url_query = f"FREQ=='{frequency}'"
url_query = url_query + f" & REF_AREA=='{country}'" if country else url_query
@@ -107,17 +113,16 @@ class OECDSTIRFetcher(Fetcher[OECDSTIRQueryParams, List[OECDSTIRData]]):
)
data["country"] = data["country"].map(CODE_TO_COUNTRY_IR)
data = data.fillna("N/A").replace("N/A", None)
- data = data.to_dict(orient="records")
-
- start_date = query.start_date.strftime("%Y-%m-%d") # type: ignore
- end_date = query.end_date.strftime("%Y-%m-%d") # type: ignore
- data = list(filter(lambda x: start_date <= x["date"] <= end_date, data))
+ data["date"] = data["date"].apply(helpers.oecd_date_to_python_date)
+ data = data[
+ (data["date"] <= query.end_date) & (data["date"] >= query.start_date)
+ ]
- return data
+ return data.to_dict(orient="records")
@staticmethod
def transform_data(
- query: OECDSTIRQueryParams, data: Dict, **kwargs: Any
+ query: OECDSTIRQueryParams, data: List[Dict], **kwargs: Any
) -> List[OECDSTIRData]:
"""Transform the data from the OECD endpoint."""
return [OECDSTIRData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/oecd/openbb_oecd/models/unemployment.py b/openbb_platform/providers/oecd/openbb_oecd/models/unemployment.py
index 406e5d500f1..b85098c69e0 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/models/unemployment.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/models/unemployment.py
@@ -104,7 +104,7 @@ class OECDUnemploymentFetcher(
query: OECDUnemploymentQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
- ) -> Dict:
+ ) -> List[Dict]:
"""Return the raw data from the OECD endpoint."""
sex = {"total": "_T", "male": "M", "female": "F"}[query.sex]
frequency = query.frequency[0].upper()
@@ -121,8 +121,20 @@ class OECDUnemploymentFetcher(
if query.country == "all"
else COUNTRY_TO_CODE_UNEMPLOYMENT[query.country]
)
- url = "https://sdmx.oecd.org/public/rest/data/OECD.SDD.TPS,DSD_LFS@DF_IALFS_INDIC,1.0/.UNE_LF........"
- data = helpers.get_possibly_cached_data(url, function="economy_unemployment")
+ # For caching, include this in the key
+ query_dict = {
+ k: v
+ for k, v in query.__dict__.items()
+ if k not in ["start_date", "end_date"]
+ }
+
+ url = (
+ f"https://sdmx.oecd.org/public/rest/data/OECD.SDD.TPS,DSD_LFS@DF_IALFS_INDIC,"
+ f"1.0/{country}.UNE_LF...{seasonal_adjustment}.{sex}.{age}..."
+ )
+ data = helpers.get_possibly_cached_data(
+ url, function="economy_unemployment", query_dict=query_dict
+ )
url_query = f"AGE=='{age}' & SEX=='{sex}' & FREQ=='{frequency}' & ADJUSTMENT=='{seasonal_adjustment}'"
url_query = url_query + f" & REF_AREA=='{country}'" if country else url_query
# Filter down
@@ -135,17 +147,17 @@ class OECDUnemploymentFetcher(
)
data["country"] = data["country"].map(CODE_TO_COUNTRY_UNEMPLOYMENT)
- data = data.to_dict(orient="records")
- start_date = query.start_date.strftime("%Y-%m-%d") # type: ignore
- end_date = query.end_date.strftime("%Y-%m-%d") # type: ignore
- data = list(filter(lambda x: start_date <= x["date"] <= end_date, data))
+ data["date"] = data["date"].apply(helpers.oecd_date_to_python_date)
+ data = data[
+ (data["date"] <= query.end_date) & (data["date"] >= query.start_date)
+ ]
- return data
+ return data.to_dict(orient="records")
# pylint: disable=unused-argument
@staticmethod
def transform_data(
- query: OECDUnemploymentQueryParams, data: Dict, **kwargs: Any
+ query: OECDUnemploymentQueryParams, data: List[Dict], **kwargs: Any
) -> List[OECDUnemploymentData]:
"""Transform the data from the OECD endpoint."""
return [OECDUnemploymentData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/oecd/openbb_oecd/utils/helpers.py b/openbb_platform/providers/oecd/openbb_oecd/utils/helpers.py
index 6533949c3ff..61e5475137d 100644
--- a/openbb_platform/providers/oecd/openbb_oecd/utils/helpers.py
+++ b/openbb_platform/providers/oecd/openbb_oecd/utils/helpers.py
@@ -2,14 +2,14 @@ import ssl
from datetime import date
from io import StringIO
from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Union
import requests
import urllib3
from defusedxml.ElementTree import fromstring
from openbb_core.app.utils import get_user_cache_directory
from openbb_core.provider import helpers
-from pandas import DataFrame, read_csv, read_parquet
+from pandas import DataFrame, read_csv, read_parquet, to_datetime
cache = get_user_cache_directory() + "/oecd"
# Create the cache directory if it does not exist
@@ -125,13 +125,13 @@ def parse_url(url: str) -> DataFrame:
return oecd_xml_to_df(response.text)
-def check_cache_exists_and_valid(function: str, cache_method: str = "csv") -> bool:
+def check_cache_exists_and_valid(cache_str: str, cache_method: str = "csv") -> bool:
"""Check if the cache exists and is valid.
Parameters
----------
- function : str
- The name of the function for which the cache is being checked.
+ cache_str : str
+ The base cache to check for.
cache_method : str, optional
The method used for caching (default is 'csv').
@@ -145,8 +145,8 @@ def check_cache_exists_and_valid(function: str, cache_method: str = "csv") -> bo
if cache_method not in ["csv", "parquet"]:
raise NotImplementedError("Currently only working with parquet or csv")
# First check that the cache exists. This will be a parquet/csv and a timestamp
- cache_path = f"{cache}/{function}.{cache_method}"
- time_cache_path = f"{cache}/{function}.timestamp"
+ cache_path = f"{cache_str}.{cache_method}"
+ time_cache_path = f"{cache_str}.timestamp"
if Path(cache_path).exists() and Path(time_cache_path).exists():
# Now check that the cache is valid. I am going to check that we write to a file the date the cache was made
# Read the timestamp
@@ -159,13 +159,13 @@ def check_cache_exists_and_valid(function: str, cache_method: str = "csv") -> bo
return False
-def write_to_cache(function: str, data: DataFrame, cache_method: str) -> None:
+def write_to_cache(cache_str: str, data: DataFrame, cache_method: str) -> None:
"""Write data to the cache.
Parameters
----------
- function : str
- The name of the function for which data is being cached.
+ cache_str : str
+ The cache key to write
data : DataFrame
The DataFrame to be cached.
cache_method : str
@@ -177,23 +177,34 @@ def write_to_cache(function: str, data: DataFrame, cache_method: str) -> None:
If the cache_method is not 'parquet'.
"""
if cache_method == "parquet":
- cache_path = f"{cache}/{function}.parquet"
+ cache_path = f"{cache_str}.parquet"
data.to_parquet(cache_path, engine="pyarrow")
# Write the current date to a file called cache/function.timestamp
- with open(f"{cache}/{function}.timestamp", "w") as f:
+ with open(f"{cache_str}.timestamp", "w") as f:
f.write(str(date.today()))
elif cache_method == "csv":
- cache_path = f"{cache}/{function}.csv"
+ cache_path = f"{cache_str}.csv"
data.to_csv(cache_path)
# Write the current date to a file called cache/function.timestamp
- with open(f"{cache}/{function}.timestamp", "w") as f:
+ with open(f"{cache_str}.timestamp", "w") as f:
f.write(str(date.today()))
else:
raise NotImplementedError
+def query_dict_to_path(query_dict: dict) -> str:
+ """Convert the query dict into something usable for writing file"""
+ items = sorted(query_dict.items())
+ key_parts = [f"{key}_{value}" for key, value in items]
+ return "-".join(key_parts).replace("/", "_").replace(" ", "_")
+
+
def get_possibly_cached_data(
- url: str, function: Optional[str] = None, cache_method: str = "csv"
+ url: str,
+ function: Optional[str] = None,
+ query_dict: Optional[dict] = None,
+ cache_method: str = "csv",
+ skip_cache: bool = False,
) -> DataFrame:
"""
Retrieve data from a given URL or from the cache if available and valid.
@@ -204,6 +215,8 @@ def get_possibly_cached_data(
The URL from which to fetch the data if it's not available in the cache.
function : Optional[str], optional
The name of the function for which data is being fetched or cached.
+ query_dict : Optional[dict], optional
+ A dictionary containing the query parameters for the function.
cache_method : str, optional
The method used for caching the data (default is 'csv').
@@ -212,20 +225,35 @@ def get_possibly_cached_data(
DataFrame
A Pandas DataFrame containing the fetched or cached data.
"""
+
+ base_cache = f"{cache}/{function}_{query_dict_to_path(query_dict)}"
if cache_method == "parquet":
- cache_path = f"{cache}/{function}.parquet"
+ cache_path = base_cache + ".parquet"
elif cache_method == "csv":
- cache_path = f"{cache}/{function}.csv"
+ cache_path = base_cache + ".csv"
use_cache = check_cache_exists_and_valid(
- function=function, cache_method=cache_method
+ cache_str=base_cache, cache_method=cache_method
)
- if use_cache:
+ if use_cache and not skip_cache:
if cache_method == "csv":
data = read_csv(cache_path)
elif cache_method == "parquet":
data = read_parquet(cache_path, engine="pyarrow")
else:
data = parse_url(url)
- write_to_cache(function=function, data=data, cache_method=cache_method)
+ if not skip_cache:
+ write_to_cache(cache_str=base_cache, data=data, cache_method=cache_method)
return data
+
+
+def oecd_date_to_python_date(input_date: Union[str, int]) -> date:
+ """Darrens good idea to make the dates filterable"""
+ input_date = str(input_date)
+ if "Q" in input_date:
+ return to_datetime(input_date).to_period("Q").to_timestamp("Q").date()
+ if len(input_date) == 4:
+ return date(int(input_date), 12, 31)
+ if len(input_date) == 7:
+ return to_datetime(input_date).to_period("M").to_timestamp("M").date()
+ raise ValueError("Date not in expected format")
diff --git a/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_forecast_gdp_fetcher.yaml b/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_forecast_gdp_fetcher.yaml
index d92e81bb321..5f82f5b0a10 100644
--- a/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_forecast_gdp_fetcher.yaml
+++ b/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_forecast_gdp_fetcher.yaml
@@ -81,13 +81,13 @@ interactions:
Cache-Control:
- private
Content-Disposition:
- - attachment;filename=DP_LIVE_26012024223440840.csv
+ - attachment;filename=DP_LIVE_29022024203625920.csv
Content-Encoding:
- gzip
Content-Type:
- text/csv
Date:
- - Fri, 26 Jan 2024 21:34:39 GMT
+ - Thu, 29 Feb 2024 19:36:25 GMT
Transfer-Encoding:
- chunked
Vary:
diff --git a/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_nominal_gdp_fetcher.yaml b/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_nominal_gdp_fetcher.yaml
index 7ca5b45cb4d..be7d8beb160 100644
--- a/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_nominal_gdp_fetcher.yaml
+++ b/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_nominal_gdp_fetcher.yaml
@@ -75,13 +75,13 @@ interactions:
Cache-Control:
- private
Content-Disposition:
- - attachment;filename=DP_LIVE_26012024223438996.csv
+ - attachment;filename=DP_LIVE_29022024203624123.csv
Content-Encoding:
- gzip
Content-Type:
- text/csv
Date:
- - Fri, 26 Jan 2024 21:34:38 GMT
+ - Thu, 29 Feb 2024 19:36:23 GMT
Transfer-Encoding:
- chunked
Vary:
diff --git a/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_real_gdp_fetcher.yaml b/openbb_platform/providers/oecd/tests/record/http/test_oecd_fetchers/test_oecd_real_gdp_fetcher.yaml
index 83ad10005d5..773b61fa764 100644
--- a/openbb_platfo