[BugFix] Fix World News (#6064)

* fix fmp

* fix intrinio

* fix biztoc

* actually commit intrinio fix

* fix tiingo

* unused import

* fix tests

* black

* ruff

* benzinga empty strings

* codespell

* fix test?

* more empty strings

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>

12 files changed, 80 insertions, 45 deletions

diff --git a/openbb_platform/core/openbb_core/provider/standard_models/world_news.py b/openbb_platform/core/openbb_core/provider/standard_models/world_news.py
index 36949c754cc..085cb82679a 100644
--- a/openbb_platform/core/openbb_core/provider/standard_models/world_news.py
+++ b/openbb_platform/core/openbb_core/provider/standard_models/world_news.py
@@ -21,9 +21,9 @@ class WorldNewsQueryParams(QueryParams):
     """World News Query."""
 
     limit: NonNegativeInt = Field(
-        default=20,
+        default=2500,
         description=QUERY_DESCRIPTIONS.get("limit", "")
-        + " Here its the no. of articles to return.",
+        + " The number of articles to return.",
     )
     start_date: Optional[dateType] = Field(
         default=None, description=QUERY_DESCRIPTIONS.get("start_date", "")
@@ -38,7 +38,7 @@ class WorldNewsQueryParams(QueryParams):
         """Populate start date if empty."""
         if not v:
             now = datetime.now().date()
-            v = now - relativedelta(years=1)
+            v = now - relativedelta(weeks=2)
         return v
 
     @field_validator("end_date", mode="before")
@@ -55,11 +55,11 @@ class WorldNewsData(Data):
 
     date: datetime = Field(
         description=DATA_DESCRIPTIONS.get("date", "")
-        + " Here it is the published date of the news."
+        + " The published date of the article."
     )
-    title: str = Field(description="Title of the news.")
+    title: str = Field(description="Title of the article.")
     images: Optional[List[Dict[str, str]]] = Field(
-        default=None, description="Images associated with the news."
+        default=None, description="Images associated with the article."
     )
-    text: Optional[str] = Field(default=None, description="Text/body of the news.")
-    url: Optional[str] = Field(default=None, description="URL of the news.")
+    text: Optional[str] = Field(default=None, description="Text/body of the article.")
+    url: Optional[str] = Field(default=None, description="URL to the article.")
diff --git a/openbb_platform/extensions/news/integration/test_news_api.py b/openbb_platform/extensions/news/integration/test_news_api.py
index abb497ae09d..5b15455b937 100644
--- a/openbb_platform/extensions/news/integration/test_news_api.py
+++ b/openbb_platform/extensions/news/integration/test_news_api.py
@@ -76,6 +76,7 @@ def headers():
                 "source": "bloomberg.com",
                 "start_date": None,
                 "end_date": None,
+                "offset": 0,
             }
         ),
     ],
diff --git a/openbb_platform/extensions/news/integration/test_news_python.py b/openbb_platform/extensions/news/integration/test_news_python.py
index b54745e1d2c..755e51109ba 100644
--- a/openbb_platform/extensions/news/integration/test_news_python.py
+++ b/openbb_platform/extensions/news/integration/test_news_python.py
@@ -75,6 +75,7 @@ def obb(pytestconfig):  # pylint: disable=inconsistent-return-statements
                 "source": "bloomberg.com",
                 "start_date": None,
                 "end_date": None,
+                "offset": 0,
             }
         ),
     ],
diff --git a/openbb_platform/providers/benzinga/openbb_benzinga/models/world_news.py b/openbb_platform/providers/benzinga/openbb_benzinga/models/world_news.py
index 64a312a6aa1..0e669771a02 100644
--- a/openbb_platform/providers/benzinga/openbb_benzinga/models/world_news.py
+++ b/openbb_platform/providers/benzinga/openbb_benzinga/models/world_news.py
@@ -1,5 +1,7 @@
 """Benzinga World News Model."""
 
+# pylint: disable=unused-argument
+
 import math
 from datetime import (
     date as dateType,
@@ -97,21 +99,31 @@ class BenzingaWorldNewsData(WorldNewsData):
     )
 
     @field_validator("date", "updated", mode="before", check_fields=False)
-    def date_validate(cls, v):  # pylint: disable=E0213
+    @classmethod
+    def date_validate(cls, v):
         """Return the date as a datetime object."""
         return datetime.strptime(v, "%a, %d %b %Y %H:%M:%S %z")
 
     @field_validator("stocks", "channels", "tags", mode="before", check_fields=False)
-    def list_validate(cls, v):  # pylint: disable=E0213
+    @classmethod
+    def list_validate(cls, v):
         """Return the list as a string."""
-        return ",".join(
-            [item.get("name", None) for item in v if item.get("name", None)]
-        )
+        v = ",".join([item.get("name", None) for item in v if item.get("name", None)])
+        return v if v != "" else None
+
+    @field_validator(
+        "id", "text", "teaser", "title", "author", mode="before", check_fields=False
+    )
+    @classmethod
+    def id_validate(cls, v):
+        """Return the a string if the field is not empty."""
+        return str(v) if v else None
 
-    @field_validator("id", mode="before", check_fields=False)
-    def id_validate(cls, v):  # pylint: disable=E0213
-        """Return the id as a string."""
-        return str(v)
+    @field_validator("images", mode="before", check_fields=False)
+    @classmethod
+    def empty_list(cls, v):
+        """Return None instead of []"""
+        return None if v == [] else v
 
 
 class BenzingaWorldNewsFetcher(
@@ -132,7 +144,7 @@ class BenzingaWorldNewsFetcher(
         query: BenzingaWorldNewsQueryParams,
         credentials: Optional[Dict[str, str]],
         **kwargs: Any,
-    ) -> List[dict]:
+    ) -> List[Dict]:
         """Extract the data."""
         token = credentials.get("benzinga_api_key") if credentials else ""
         base_url = "https://api.benzinga.com/api/v2/news"
@@ -151,11 +163,10 @@ class BenzingaWorldNewsFetcher(
 
         return data[: query.limit]
 
-    # pylint: disable=unused-argument
     @staticmethod
     def transform_data(
         query: BenzingaWorldNewsQueryParams,
-        data: List[dict],
+        data: List[Dict],
         **kwargs: Any,
     ) -> List[BenzingaWorldNewsData]:
         """Transform the data."""
diff --git a/openbb_platform/providers/benzinga/tests/test_benzinga_fetchers.py b/openbb_platform/providers/benzinga/tests/test_benzinga_fetchers.py
index d7658ed4b94..562280ad262 100644
--- a/openbb_platform/providers/benzinga/tests/test_benzinga_fetchers.py
+++ b/openbb_platform/providers/benzinga/tests/test_benzinga_fetchers.py
@@ -21,7 +21,7 @@ def vcr_config():
 
 @pytest.mark.record_http
 def test_benzinga_world_news_fetcher(credentials=test_credentials):
-    params = {}
+    params = {"limit": 20}
 
     fetcher = BenzingaWorldNewsFetcher()
     result = fetcher.test(params, credentials)
diff --git a/openbb_platform/providers/biztoc/openbb_biztoc/models/world_news.py b/openbb_platform/providers/biztoc/openbb_biztoc/models/world_news.py
index b1074d4e5bb..8d6d3ae2ea9 100644
--- a/openbb_platform/providers/biztoc/openbb_biztoc/models/world_news.py
+++ b/openbb_platform/providers/biztoc/openbb_biztoc/models/world_news.py
@@ -1,5 +1,6 @@
 """Biztoc World News Model."""
 
+import warnings
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Literal, Optional
 
@@ -9,10 +10,11 @@ from openbb_core.provider.standard_models.world_news import (
     WorldNewsData,
     WorldNewsQueryParams,
 )
-from openbb_core.provider.utils.helpers import filter_by_dates
 from pandas import to_datetime
 from pydantic import Field, field_validator
 
+_warn = warnings.warn
+
 
 class BiztocWorldNewsQueryParams(WorldNewsQueryParams):
     """Biztoc World News Query."""
@@ -37,7 +39,12 @@ class BiztocWorldNewsQueryParams(WorldNewsQueryParams):
 class BiztocWorldNewsData(WorldNewsData):
     """Biztoc World News Data."""
 
-    __alias_dict__ = {"date": "created", "text": "body", "site": "domain"}
+    __alias_dict__ = {
+        "date": "created",
+        "text": "body",
+        "site": "domain",
+        "images": "img",
+    }
 
     images: Optional[Dict[str, str]] = Field(
         description="Images for the article.", alias="images", default=None
@@ -69,6 +76,8 @@ class BiztocWorldNewsFetcher(
     @staticmethod
     def transform_query(params: Dict[str, Any]) -> BiztocWorldNewsQueryParams:
         """Transform the query."""
+        if params.get("start_date") or params.get("end_date"):
+            _warn("start_date and end_date are not supported for this endpoint.")
         return BiztocWorldNewsQueryParams(**params)
 
     @staticmethod
@@ -96,7 +105,7 @@ class BiztocWorldNewsFetcher(
                     hours=times[item["created"]]
                 )
 
-        return data
+        return sorted(data, key=lambda x: x["created"], reverse=True)
 
     # pylint: disable=unused-argument
     @staticmethod
@@ -104,5 +113,4 @@ class BiztocWorldNewsFetcher(
         query: BiztocWorldNewsQueryParams, data: List[Dict], **kwargs: Any
     ) -> List[BiztocWorldNewsData]:
         """Transform the data to the standard format."""
-        modeled_data = [BiztocWorldNewsData.model_validate(d) for d in data]
-        return filter_by_dates(modeled_data, query.start_date, query.end_date)
+        return [BiztocWorldNewsData.model_validate(d) for d in data[: query.limit]]
diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/world_news.py b/openbb_platform/providers/fmp/openbb_fmp/models/world_news.py
index 9acdf62fe8c..4651779a09c 100644
--- a/openbb_platform/providers/fmp/openbb_fmp/models/world_news.py
+++ b/openbb_platform/providers/fmp/openbb_fmp/models/world_news.py
@@ -1,6 +1,7 @@
 """FMP World News Model."""
 
 import math
+import warnings
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
@@ -9,9 +10,11 @@ from openbb_core.provider.standard_models.world_news import (
     WorldNewsData,
     WorldNewsQueryParams,
 )
-from openbb_core.provider.utils.helpers import amake_requests, filter_by_dates
+from openbb_core.provider.utils.helpers import amake_requests
 from pydantic import Field, field_validator
 
+_warn = warnings.warn
+
 
 class FMPWorldNewsQueryParams(WorldNewsQueryParams):
     """FMP World News Query.
@@ -32,6 +35,13 @@ class FMPWorldNewsData(WorldNewsData):
         """Return the date as a datetime object."""
         return datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%fZ")
 
+    @field_validator("images", mode="before", check_fields=False)
+    def images_validate(cls, v):  # pylint: disable=E0213
+        """Conform the response to a list."""
+        if isinstance(v, str):
+            return [{"o": v}]
+        return v if isinstance(v, List[Dict]) else None
+
 
 class FMPWorldNewsFetcher(
     Fetcher[
@@ -44,6 +54,8 @@ class FMPWorldNewsFetcher(
     @staticmethod
     def transform_query(params: Dict[str, Any]) -> FMPWorldNewsQueryParams:
         """Transform the query params."""
+        if params.get("start_date") or params.get("end_date"):
+            _warn("start_date and end_date are not supported for this endpoint.")
         return FMPWorldNewsQueryParams(**params)
 
     @staticmethod
@@ -62,7 +74,6 @@ class FMPWorldNewsFetcher(
             f"{base_url}/general_news?page={page}&apikey={api_key}"
             for page in range(pages)
         ]
-
         response = await amake_requests(urls, **kwargs)
         data = sorted(response, key=lambda x: x["publishedDate"], reverse=True)
 
@@ -74,9 +85,4 @@ class FMPWorldNewsFetcher(
         query: FMPWorldNewsQueryParams, data: List[Dict], **kwargs: Any
     ) -> List[FMPWorldNewsData]:
         """Return the transformed data."""
-        for d in data:
-            if isinstance(d["image"], str):
-                d["image"] = [{"url": d["image"]}]
-
-        modeled_data = [FMPWorldNewsData.model_validate(d) for d in data]
-        return filter_by_dates(modeled_data, query.start_date, query.end_date)
+        return [FMPWorldNewsData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py
index bb2c84196e9..3e82104b2c0 100644
--- a/openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py
+++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py
@@ -1,5 +1,6 @@
 """Intrinio World News Model."""
 
+import warnings
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
@@ -8,10 +9,11 @@ from openbb_core.provider.standard_models.world_news import (
     WorldNewsData,
     WorldNewsQueryParams,
 )
-from openbb_core.provider.utils.helpers import filter_by_dates
 from openbb_intrinio.utils.helpers import get_data_many
 from pydantic import Field, field_validator
 
+_warn = warnings.warn
+
 
 class IntrinioWorldNewsQueryParams(WorldNewsQueryParams):
     """Intrinio World News Query.
@@ -47,6 +49,8 @@ class IntrinioWorldNewsFetcher(
     @staticmethod
     def transform_query(params: Dict[str, Any]) -> IntrinioWorldNewsQueryParams:
         """Transform the query params."""
+        if params.get("start_date") or params.get("end_date"):
+            _warn("start_date and end_date are not supported for this endpoint.")
         return IntrinioWorldNewsQueryParams(**params)
 
     @staticmethod
@@ -69,5 +73,4 @@ class IntrinioWorldNewsFetcher(
         query: IntrinioWorldNewsQueryParams, data: List[Dict], **kwargs: Any
     ) -> List[IntrinioWorldNewsData]:
         """Return the transformed data."""
-        modeled_data = [IntrinioWorldNewsData.model_validate(d) for d in data]
-        return filter_by_dates(modeled_data, query.start_date, query.end_date)
+        return [IntrinioWorldNewsData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_world_news_fetcher.yaml b/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_world_news_fetcher.yaml
index 66d3723ab93..c15ce83999b 100644
--- a/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_world_news_fetcher.yaml
+++ b/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_world_news_fetcher.yaml
@@ -9,7 +9,7 @@ interactions:
       Connection:
       - keep-alive
     method: GET
-    uri: https://api-v2.intrinio.com/companies/news?api_key=MOCK_API_KEY&page_size=20
+    uri: https://api-v2.intrinio.com/companies/news?page_size=2500&api_key=MOCK_API_KEY
   response:
     body:
       string: !!binary |
diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/world_news.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/world_news.py
index c5f0e657331..486e2a76bf9 100644
--- a/openbb_platform/providers/tiingo/openbb_tiingo/models/world_news.py
+++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/world_news.py
@@ -9,11 +9,10 @@ from openbb_core.provider.standard_models.world_news import (
     WorldNewsData,
     WorldNewsQueryParams,
 )
-from openbb_core.provider.utils.helpers import filter_by_dates, get_querystring
+from openbb_core.provider.utils.helpers import get_querystring
+from openbb_tiingo.utils.helpers import get_data_many
 from pydantic import Field, field_validator
 
-from ..utils.helpers import get_data_many
-
 
 class TiingoWorldNewsQueryParams(WorldNewsQueryParams):
     """Tiingo World News Query.
@@ -21,6 +20,13 @@ class TiingoWorldNewsQueryParams(WorldNewsQueryParams):
     Source: https://www.tiingo.com/documentation/news
     """
 
+    __alias_dict__ = {
+        "start_date": "startDate",
+        "end_date": "endDate",
+    }
+    offset: Optional[int] = Field(
+        default=0, description="Page offset, used in conjunction with limit."
+    )
     source: Optional[str] = Field(
         default=None, description="A comma-separated list of the domains requested."
     )
@@ -95,5 +101,4 @@ class TiingoWorldNewsFetcher(
         query: TiingoWorldNewsQueryParams, data: List[Dict], **kwargs: Any
     ) -> List[TiingoWorldNewsData]:
         """Return the transformed data."""
-        modeled_data = [TiingoWorldNewsData.model_validate(d) for d in data]
-        return filter_by_dates(modeled_data, query.start_date, query.end_date)
+        return [TiingoWorldNewsData.model_validate(d) for d in data]
diff --git a/openbb_platform/providers/tiingo/tests/record/http/test_tiingo_fetchers/test_tiingo_world_news_fetcher.yaml b/openbb_platform/providers/tiingo/tests/record/http/test_tiingo_fetchers/test_tiingo_world_news_fetcher.yaml
index 69118501897..5c1588a3faf 100644
--- a/openbb_platform/providers/tiingo/tests/record/http/test_tiingo_fetchers/test_tiingo_world_news_fetcher.yaml
+++ b/openbb_platform/providers/tiingo/tests/record/http/test_tiingo_fetchers/test_tiingo_world_news_fetcher.yaml
@@ -9,7 +9,7 @@ interactions:
       Connection:
       - keep-alive
     method: GET
-    uri: https://api.tiingo.com/tiingo/news?limit=20&token=MOCK_TOKEN
+    uri: https://api.tiingo.com/tiingo/news?limit=20&offset=0&token=MOCK_TOKEN
   response:
     body:
       string: "[{\"id\":59591461,\"publishedDate\":\"2023-11-16T08:38:56Z\",\"title\":\"How
diff --git a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py
index 72929120006..24357f9e02e 100644
--- a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py
+++ b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py
@@ -48,7 +48,7 @@ def test_tiingo_company_news_fetcher(credentials=test_credentials):
 
 @pytest.mark.record_http
 def test_tiingo_world_news_fetcher(credentials=test_credentials):
-    params = {}
+    params = {"limit": 20}
 
     fetcher = TiingoWorldNewsFetcher()
     result = fetcher.test(params, credentials)