[BugFix] Fix empty docstring in data processing functions (#6206)

* update etf static file

update reference.json with changes

* data processing docstrings can be empty

use self._write func to create reference.json file

3 files changed, 3838 insertions, 21816 deletions

diff --git a/openbb_platform/core/openbb_core/app/static/package_builder.py b/openbb_platform/core/openbb_core/app/static/package_builder.py
index 9de60e14837..a784e5e8164 100644
--- a/openbb_platform/core/openbb_core/app/static/package_builder.py
+++ b/openbb_platform/core/openbb_core/app/static/package_builder.py
@@ -8,7 +8,7 @@ import shutil
 import sys
 from dataclasses import Field
 from inspect import Parameter, _empty, isclass, signature
-from json import dump, dumps, load
+from json import dumps, load
 from pathlib import Path
 from typing import (
     Any,
@@ -189,11 +189,8 @@ class PackageBuilder:
         """Save the reference.json file."""
         self.console.log("\nWriting reference file...")
         data = ReferenceGenerator.get_reference_data()
-        file_path = self.directory / "assets" / "reference.json"
-        # Dumping the reference dictionary as a JSON file
-        self.console.log(str(file_path))
-        with open(file_path, "w", encoding="utf-8") as f:
-            dump(data, f, indent=4)
+        code = dumps(obj=data, indent=4)
+        self._write(code=code, name="reference", extension="json", folder="assets")
 
     def _run_linters(self):
         """Run the linters."""
@@ -1392,6 +1389,8 @@ class ReferenceGenerator:
                 List of dictionaries containing the name,type, description, default
                 and optionality of each parameter.
         """
+        parameters_list = []
+
         # Define a regex pattern to match parameter blocks
         # This pattern looks for a parameter name followed by " : ", then captures the type and description
         pattern = re.compile(
@@ -1401,33 +1400,31 @@ class ReferenceGenerator:
         # Find all matches in the docstring
         matches = pattern.finditer(docstring)
 
-        # Initialize an empty list to store parameter dictionaries
-        parameters_list = []
-
-        # Iterate over the matches to extract details
-        for match in matches:
-            # Extract named groups as a dictionary
-            param_info = match.groupdict()
+        if matches:
+            # Iterate over the matches to extract details
+            for match in matches:
+                # Extract named groups as a dictionary
+                param_info = match.groupdict()
 
-            # Determine if the parameter is optional
-            is_optional = "Optional" in param_info["type"]
+                # Determine if the parameter is optional
+                is_optional = "Optional" in param_info["type"]
 
-            # If no default value is captured, set it to an empty string
-            default_value = (
-                param_info["default"] if param_info["default"] is not None else ""
-            )
+                # If no default value is captured, set it to an empty string
+                default_value = (
+                    param_info["default"] if param_info["default"] is not None else ""
+                )
 
-            # Create a new dictionary with fields in the desired order
-            param_dict = {
-                "name": param_info["name"],
-                "type": param_info["type"],
-                "description": param_info["description"],
-                "default": default_value,
-                "optional": is_optional,
-            }
+                # Create a new dictionary with fields in the desired order
+                param_dict = {
+                    "name": param_info["name"],
+                    "type": param_info["type"],
+                    "description": param_info["description"],
+                    "default": default_value,
+                    "optional": is_optional,
+                }
 
-            # Append the dictionary to the list
-            parameters_list.append(param_dict)
+                # Append the dictionary to the list
+                parameters_list.append(param_dict)
 
         return parameters_list
 
@@ -1445,23 +1442,27 @@ class ReferenceGenerator:
             Dict[str, str]:
                 Dictionary containing the name, type, description of the return value
         """
+        return_info = ""
+
         # Define a regex pattern to match the Returns section
         # This pattern captures the model name inside "OBBject[]" and its description
         match = re.search(r"Returns\n\s*-------\n\s*([^\n]+)\n\s*([^\n]+)", docstring)
-        return_type = match.group(1).strip()  # type: ignore
-        # Remove newlines and indentation from the description
-        description = match.group(2).strip().replace("\n", "").replace("    ", "")  # type: ignore
-        # Adjust regex to correctly capture content inside brackets, including nested brackets
-        content_inside_brackets = re.search(
-            r"OBBject\[\s*((?:[^\[\]]|\[[^\[\]]*\])*)\s*\]", return_type
-        )
-        return_type_content = content_inside_brackets.group(1)  # type: ignore
 
-        return_info = (
-            f"OBBject\n"
-            f"{create_indent(1)}results : {return_type_content}\n"
-            f"{create_indent(2)}{description}"
-        )
+        if match:
+            return_type = match.group(1).strip()  # type: ignore
+            # Remove newlines and indentation from the description
+            description = match.group(2).strip().replace("\n", "").replace("    ", "")  # type: ignore
+            # Adjust regex to correctly capture content inside brackets, including nested brackets
+            content_inside_brackets = re.search(
+                r"OBBject\[\s*((?:[^\[\]]|\[[^\[\]]*\])*)\s*\]", return_type
+            )
+            return_type_content = content_inside_brackets.group(1)  # type: ignore
+
+            return_info = (
+                f"OBBject\n"
+                f"{create_indent(1)}results : {return_type_content}\n"
+                f"{create_indent(2)}{description}"
+            )
 
         return return_info
 
@@ -1557,21 +1558,18 @@ class ReferenceGenerator:
                 # POST method router `description` attribute is unreliable as it may or
                 # may not contain the "Parameters" and "Returns" sections. Hence, the
                 # endpoint function docstring is used instead.
-                description = route_func.__doc__.split("Parameters")[0].strip()  # type: ignore
+                docstring = getattr(route_func, "__doc__", "")
+                description = docstring.split("Parameters")[0].strip()
                 # Remove extra spaces in between the string
                 reference[path]["description"] = re.sub(" +", " ", description)
                 # Add endpoint parameters fields for POST methods
-                reference[path]["parameters"][
-                    "standard"
-                ] = ReferenceGenerator.get_post_method_parameters_info(
-                    route_func.__doc__  # type: ignore
+                reference[path]["parameters"]["standard"] = (
+                    ReferenceGenerator.get_post_method_parameters_info(docstring)
                 )
                 # Add endpoint returns data
                 # Currently only OBBject object is returned
-                reference[path]["returns"][
-                    "OBBject"
-                ] = cls.get_post_method_returns_info(
-                    route_func.__doc__  # type: ignore
+                reference[path]["returns"]["OBBject"] = (
+                    cls.get_post_method_returns_info(docstring)
                 )
 
         return reference
diff --git a/openbb_platform/openbb/assets/reference.json b/openbb_platform/openbb/assets/reference.json
index 623bd113d81..465a0f4bb6f 100644
--- a/openbb_platform/openbb/assets/reference.json
+++ b/openbb_platform/openbb/assets/reference.json
@@ -13,224 +13,85 @@
                     "type": "Union[str, List[str]]",
                     "description": "Symbol to get data for. Can use CURR1-CURR2 or CURR1CURR2 format. Multiple items allowed for provider(s): fmp, polygon, tiingo, yfinance.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "start_date",
                     "type": "Union[date, str]",
                     "description": "Start date of the data, in YYYY-MM-DD format.",
                     "default": "None",
-                    "optional": true,
-                    "standard": true
+                    "optional": true
                 },
                 {
                     "name": "end_date",
                     "type": "Union[date, str]",
                     "description": "End date of the data, in YYYY-MM-DD format.",
                     "default": "None",
-                    "optional": true,
-                    "standard": true
+                    "optional": true
                 },
                 {
                     "name": "provider",
                     "type": "Literal['fmp', 'polygon', 'tiingo', 'yfinance']",
                     "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default.",
                     "default": "fmp",
-                    "optional": true,
-                    "standard": true
+                    "optional": true
                 }
             ],
             "fmp": [
                 {
-                    "name": "symbol",
-                    "type": "Union[str, List[str]]",
-                    "description": "Symbol to get data for. Can use CURR1-CURR2 or CURR1CURR2 format. Multiple items allowed for provider(s): fmp, polygon, tiingo, yfinance.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "start_date",
-                    "type": "Union[date, str]",
-                    "description": "Start date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "end_date",
-                    "type": "Union[date, str]",
-                    "description": "End date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "provider",
-                    "type": "Literal['fmp', 'polygon', 'tiingo', 'yfinance']",
-                    "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default.",
-                    "default": "fmp",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "interval",
                     "type": "Literal['1m', '5m', '15m', '30m', '1h', '4h', '1d']",
                     "description": "Time interval of the data to return.",
                     "default": "1d",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
             "polygon": [
                 {
-                    "name": "symbol",
-                    "type": "Union[str, List[str]]",
-                    "description": "Symbol to get data for. Can use CURR1-CURR2 or CURR1CURR2 format. Multiple items allowed for provider(s): fmp, polygon, tiingo, yfinance.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "start_date",
-                    "type": "Union[date, str]",
-                    "description": "Start date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "end_date",
-                    "type": "Union[date, str]",
-                    "description": "End date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "provider",
-                    "type": "Literal['fmp', 'polygon', 'tiingo', 'yfinance']",
-                    "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default.",
-                    "default": "fmp",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "interval",
                     "type": "str",
                     "description": "Time interval of the data to return. The numeric portion of the interval can be any positive integer. The letter portion can be one of the following: s, m, h, d, W, M, Q, Y",
                     "default": "1d",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "sort",
                     "type": "Literal['asc', 'desc']",
                     "description": "Sort order of the data. This impacts the results in combination with the 'limit' parameter. The results are always returned in ascending order by date.",
                     "default": "asc",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "limit",
                     "type": "int",
                     "description": "The number of data entries to return.",
                     "default": "49999",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
             "tiingo": [
                 {
-                    "name": "symbol",
-                    "type": "Union[str, List[str]]",
-                    "description": "Symbol to get data for. Can use CURR1-CURR2 or CURR1CURR2 format. Multiple items allowed for provider(s): fmp, polygon, tiingo, yfinance.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "start_date",
-                    "type": "Union[date, str]",
-                    "description": "Start date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "end_date",
-                    "type": "Union[date, str]",
-                    "description": "End date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "provider",
-                    "type": "Literal['fmp', 'polygon', 'tiingo', 'yfinance']",
-                    "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default.",
-                    "default": "fmp",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "interval",
                     "type": "Literal['1m', '5m', '15m', '30m', '1h', '4h', '1d']",
                     "description": "Time interval of the data to return.",
                     "default": "1d",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "exchanges",
                     "type": "List[str]",
                     "description": "To limit the query to a subset of exchanges e.g. ['POLONIEX', 'GDAX']",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
             "yfinance": [
                 {
-                    "name": "symbol",
-                    "type": "Union[str, List[str]]",
-                    "description": "Symbol to get data for. Can use CURR1-CURR2 or CURR1CURR2 format. Multiple items allowed for provider(s): fmp, polygon, tiingo, yfinance.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "start_date",
-                    "type": "Union[date, str]",
-                    "description": "Start date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "end_date",
-                    "type": "Union[date, str]",
-                    "description": "End date of the data, in YYYY-MM-DD format.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
-                    "name": "provider",
-                    "type": "Literal['fmp', 'polygon', 'tiingo', 'yfinance']",
-                    "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default.",
-                    "default": "fmp",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "interval",
                     "type": "Literal['1m', '2m', '5m', '15m', '30m', '60m', '90m', '1h', '1d', '5d', '1W', '1M', '1Q']",
                     "description": "Time interval of the data to return.",
                     "default": "1d",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ]
         },
@@ -244,338 +105,100 @@
                     "type": "Union[date, datetime]",
                     "description": "The date of the data.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "open",
                     "type": "float",
                     "description": "The open price.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "high",
                     "type": "float",
                     "description": "The high price.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "low",
                     "type": "float",
                     "description": "The low price.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "close",
                     "type": "float",
                     "description": "The close price.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "volume",
                     "type": "float",
                     "description": "The trading volume.",
                     "default": "",
-                    "optional": false,
-                    "standard": true
+                    "optional": false
                 },
                 {
                     "name": "vwap",
                     "type": "float, Gt(gt=0)",
                     "description": "Volume Weighted Average Price over the period.",
                     "default": "None",
-                    "optional": true,
-                    "standard": true
+                    "optional": true
                 }
             ],
             "fmp": [
                 {
-                    "name": "date",
-                    "type": "Union[date, datetime]",
-                    "description": "The date of the data.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "open",
-                    "type": "float",
-                    "description": "The open price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "high",
-                    "type": "float",
-                    "description": "The high price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "low",
-                    "type": "float",
-                    "description": "The low price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "close",
-                    "type": "float",
-                    "description": "The close price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "volume",
-                    "type": "float",
-                    "description": "The trading volume.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "vwap",
-                    "type": "float, Gt(gt=0)",
-                    "description": "Volume Weighted Average Price over the period.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "adj_close",
                     "type": "float",
                     "description": "The adjusted close price.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "change",
                     "type": "float",
                     "description": "Change in the price from the previous close.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "change_percent",
                     "type": "float",
                     "description": "Change in the price from the previous close, as a normalized percent.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
             "polygon": [
                 {
-                    "name": "date",
-                    "type": "Union[date, datetime]",
-                    "description": "The date of the data.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "open",
-                    "type": "float",
-                    "description": "The open price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "high",
-                    "type": "float",
-                    "description": "The high price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "low",
-                    "type": "float",
-                    "description": "The low price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "close",
-                    "type": "float",
-                    "description": "The close price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "volume",
-                    "type": "float",
-                    "description": "The trading volume.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "vwap",
-                    "type": "float, Gt(gt=0)",
-                    "description": "Volume Weighted Average Price over the period.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "transactions",
                     "type": "int, Gt(gt=0)",
                     "description": "Number of transactions for the symbol in the time period.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
             "tiingo": [
                 {
-                    "name": "date",
-                    "type": "Union[date, datetime]",
-                    "description": "The date of the data.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "open",
-                    "type": "float",
-                    "description": "The open price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "high",
-                    "type": "float",
-                    "description": "The high price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "low",
-                    "type": "float",
-                    "description": "The low price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "close",
-                    "type": "float",
-                    "description": "The close price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "volume",
-                    "type": "float",
-                    "description": "The trading volume.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "vwap",
-                    "type": "float, Gt(gt=0)",
-                    "description": "Volume Weighted Average Price over the period.",
-                    "default": "None",
-                    "optional": true,
-                    "standard": true
-                },
-                {
                     "name": "transactions",
                     "type": "int",
                     "description": "Number of transactions for the symbol in the time period.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 },
                 {
                     "name": "volume_notional",
                     "type": "float",
                     "description": "The last size done for the asset on the specific date in the quote currency. The volume of the asset on the specific date in the quote currency.",
                     "default": "None",
-                    "optional": true,
-                    "standard": false
+                    "optional": true
                 }
             ],
-            "yfinance": [
-                {
-                    "name": "date",
-                    "type": "Union[date, datetime]",
-                    "description": "The date of the data.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "open",
-                    "type": "float",
-                    "description": "The open price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "high",
-                    "type": "float",
-                    "description": "The high price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {
-                    "name": "low",
-                    "type": "float",
-                    "description": "The low price.",
-                    "default": "",
-                    "optional": false,
-                    "standard": true
-                },
-                {