diff options
author | Pratyush Shukla <ps4534@nyu.edu> | 2024-02-27 16:50:28 +0530 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-27 11:20:28 +0000 |
commit | 577a42461d9a123b0a717603aed0fb4e306b69eb (patch) | |
tree | 2b6afa2e4ff7ebdbb7ce11a31420ea5cb5489b69 | |
parent | 225c945fab888c83273f3163caec53748f10cd94 (diff) |
[Feature] - Platform V4 Markdown Generator V2 (#6094)
* add recent `openapi.json`
* fix data types in models
* removed `openapi.json`
* set default date value as None in PolygonCurrencyPairs class
* add function to generate reference.json file
* reworked function to improve readability
* reword function to add seo metadata
* add functions for creating markdown sections
* add code to extract data card markdown title
* add additional functions for generating index and data models file
* extract first sentence of the description for cards
* code cleanup and documentation
* linting
* linting polygon models
* add openbb import statement in create_reference_markdown_examples function
* add POST method functions
* cleanup; reworked generate_reference_index_files function
* moved development section to 7th position in the sidebar
* fix POST function params default value
add standard flag to QueryParams and Data fields
cleanup
* add type expansion from package_builder.MethodDefinition
* sort data models cards alphabetically
make printing less verbose
* make MAX_CARDS global
display less content in cards in Commannds section
* cleanup
* Remove '_' from the cards under Commands section
* " to ' in econometrics/causality
* replace ' with " in ReferenceCard for reference dir index files
* remove extra . from the quantile function description
* shoutout to @deeleeramone for finding POST method description bug!
* set correct value for standard field
* handle BaseModel types in provider data fields
* unit tests for the platform markdown generator v2
* yeet 'Default' and 'Optional' columns in the 'Data' section
* last minute bug fix
* add info for multiple symbols
* make multiple items info same as platform static
* organize sections properly
* sort reference sub-directories alphabetically
* extra space in 'OBBject extra' description
* add type expansion for fields with multiple items
POST method cleanup
---------
Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
9 files changed, 1215 insertions, 673 deletions
diff --git a/openbb_platform/core/openbb_core/app/static/package_builder.py b/openbb_platform/core/openbb_core/app/static/package_builder.py index 9e75ce8fc31..abfad97eda6 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder.py @@ -843,7 +843,7 @@ class DocstringGenerator: " List of warnings.\n" " chart : Optional[Chart]\n" " Chart object.\n" - " extra: Dict[str, Any]\n" + " extra : Dict[str, Any]\n" " Extra info.\n" ) obbject_description = obbject_description.replace("NoneType", "None") diff --git a/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py b/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py index 0c7b52529a0..2a1a941e553 100644 --- a/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py +++ b/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py @@ -353,10 +353,10 @@ def causality( x_column: str, lag: PositiveInt = 3, ) -> OBBject[Data]: - """Perform Granger causality test to determine if X "causes" y. + """Perform Granger causality test to determine if X 'causes' y. The Granger causality test is a statistical hypothesis test to determine if one time series is useful in - forecasting another. While "causality" in this context does not imply a cause-and-effect relationship in + forecasting another. While 'causality' in this context does not imply a cause-and-effect relationship in the philosophical sense, it does test whether changes in one variable are systematically followed by changes in another variable, suggesting a predictive relationship. By specifying a lag, you set the number of periods to look back in the time series to assess this relationship. This test is particularly useful in economic and diff --git a/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py b/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py index 6e464083aea..c2cc5c7a1ae 100644 --- a/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py +++ b/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py @@ -230,7 +230,7 @@ def quantile( Quantiles are points dividing the range of a probability distribution into intervals with equal probabilities, or dividing the sample in the same way. This function is useful for understanding the distribution of data - within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk.. + within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk. Parameters: data: List[Data] diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py b/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py index 66adef15c48..2ea951781d2 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py @@ -26,7 +26,7 @@ class PolygonCurrencyPairsQueryParams(CurrencyPairsQueryParams): default=None, description="Symbol of the pair to search." ) date: Optional[dateType] = Field( - default=datetime.now().date(), description=QUERY_DESCRIPTIONS.get("date", "") + default=None, description=QUERY_DESCRIPTIONS.get("date", "") ) search: Optional[str] = Field( default="", @@ -159,6 +159,7 @@ class PolygonCurrencyPairsFetcher( return all_data + # pylint: disable=unused-argument @staticmethod def transform_data( query: PolygonCurrencyPairsQueryParams, diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py b/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py index ab92bc81e02..e332745c99e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py @@ -92,7 +92,7 @@ class PolygonEquityNBBOData(EquityNBBOData): conditions: Optional[Union[str, List[int], List[str]]] = Field( default=None, description="A list of condition codes.", alias="conditions" ) - indicators: Optional[List] = Field( + indicators: Optional[List[int]] = Field( default=None, description="A list of indicator codes.", alias="indicators" ) sequence_num: Optional[int] = Field( @@ -208,6 +208,7 @@ class PolygonEquityNBBOFetcher( return data + # pylint: disable=unused-argument @staticmethod def transform_data( query: PolygonEquityNBBOQueryParams, diff --git a/openbb_platform/providers/sec/openbb_sec/models/schema_files.py b/openbb_platform/providers/sec/openbb_sec/models/schema_files.py index 93ef045baf7..682d03985b1 100644 --- a/openbb_platform/providers/sec/openbb_sec/models/schema_files.py +++ b/openbb_platform/providers/sec/openbb_sec/models/schema_files.py @@ -23,7 +23,7 @@ class SecSchemaFilesQueryParams(CotSearchQueryParams): class SecSchemaFilesData(Data): """SEC Schema Files List Data.""" - files: List = Field(description="Dictionary of URLs to SEC Schema Files") + files: List[str] = Field(description="Dictionary of URLs to SEC Schema Files") class SecSchemaFilesFetcher(Fetcher[SecSchemaFilesQueryParams, SecSchemaFilesData]): diff --git a/website/content/platform/development/_category_.json b/website/content/platform/development/_category_.json index 8219ce2ae74..7a45206fc82 100644 --- a/website/content/platform/development/_category_.json +++ b/website/content/platform/development/_category_.json @@ -1,4 +1,4 @@ { "label": "Development", - "position": 6 -} + "position": 7 +}
\ No newline at end of file diff --git a/website/generate_platform_v4_markdown.py b/website/generate_platform_v4_markdown.py index c377a68f70c..66842d771f4 100644 --- a/website/generate_platform_v4_markdown.py +++ b/website/generate_platform_v4_markdown.py @@ -1,602 +1,698 @@ +"""Platform V4 Markdown Generator Script.""" + +# pylint: disable=too-many-lines + import inspect import json import re import shutil -from inspect import Parameter, _empty, signature from pathlib import Path -from textwrap import shorten -from typing import Any, Callable, Dict, List, TextIO, Tuple, Union +from typing import Any, Callable, Dict, List -from docstring_parser import parse from openbb_core.app.provider_interface import ProviderInterface -from openbb_core.app.static.package_builder import ( - DocstringGenerator, - MethodDefinition, - PathHandler, -) +from openbb_core.app.router import RouterLoader +from openbb_core.app.static.package_builder import MethodDefinition from openbb_core.provider import standard_models -from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined + +# Number of spaces to substitute tabs for indentation +TAB_WIDTH = 4 + +# Maximum number of commands to display on the cards +MAX_COMMANDS = 8 -website_path = Path(__file__).parent.absolute() -SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads( - (website_path / "metadata/platform_v4_seo_metadata.json").read_text() -) +# Paths to use for generating and storing the markdown files +WEBSITE_PATH = Path(__file__).parent.absolute() +SEO_METADATA_PATH = Path(WEBSITE_PATH / "metadata/platform_v4_seo_metadata.json") +PLATFORM_CONTENT_PATH = Path(WEBSITE_PATH / "content/platform") +PLATFORM_REFERENCE_PATH = Path(WEBSITE_PATH / "content/platform/reference") +PLATFORM_DATA_MODELS_PATH = Path(WEBSITE_PATH / "content/platform/data_models") -REFERENCE_IMPORT_UL = """import ReferenceCard from "@site/src/components/General/NewReferenceCard"; +# Imports used in the generated markdown files +PLATFORM_REFERENCE_IMPORT = "import ReferenceCard from '@site/src/components/General/NewReferenceCard';" # fmt: skip +PLATFORM_REFERENCE_UL_ELEMENT = '<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">' # noqa: E501 -<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6"> -""" -reference_import = ( - 'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n' -) -refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">""" +def get_field_data_type(field_type: Any) -> str: + """Get the implicit data type from the field type. + String manipulation is used to extract the implicit + data type from the field type. -def get_docstring_meta( - func: Callable, full_command_path: str, formatted_params: Dict[str, Parameter] -) -> Dict[str, Any]: - """Extracts the meta information from the docstring of a function with no standardized results model.""" - meta_command = {} - doc_parsed = parse(func.__doc__) # type: ignore + Args: + field_type (Any): typing object field type - cmd_params = [] - for param in doc_parsed.params: - arg_default = ( - formatted_params[param.arg_name].default - if param.arg_name in formatted_params - else None + Returns: + str: String representation of the implicit field tzxype + """ + + try: + if "BeforeValidator" in str(field_type): + field_type = "int" + + if "Optional" in str(field_type): + field_type = str(field_type.__args__[0]) + + if "Annotated[" in str(field_type): + field_type = str(field_type).rsplit("[", maxsplit=1)[-1].split(",")[0] + + if "models" in str(field_type): + field_type = str(field_type).rsplit(".", maxsplit=1)[-1] + + field_type = ( + str(field_type) + .replace("<class '", "") + .replace("'>", "") + .replace("typing.", "") + .replace("pydantic.types.", "") + .replace("openbb_core.provider.abstract.data.", "") + .replace("datetime.datetime", "datetime") + .replace("datetime.date", "date") + .replace("NoneType", "None") + .replace(", None", "") ) - cmd_params.append( + except TypeError: + field_type = str(field_type) + + return field_type + + +def get_provider_parameter_info(endpoint: Callable) -> Dict[str, str]: + """Get the name, type, description, default value and optionality + information for the provider parameter. + + Function signature is insepcted to get the parameters of the router + endpoint function. The provider parameter is then extracted from the + function type annotations then the information is extracted from it. + + Args: + endpoint (Callable): Router endpoint function + + Returns: + Dict[str, str]: Dictionary of the provider parameter information + """ + + params_dict = endpoint.__annotations__ + model_type = params_dict["provider_choices"].__args__[0] + provider_params_field = model_type.__dataclass_fields__["provider"] + + # Type is Union[Literal[<provider_name>], None] + default = provider_params_field.type.__args__[0] + description = ( + "The provider to use for the query, by default None. " + "If None, the provider specified in defaults is selected " + f"or '{default}' if there is no default." + ) + + provider_parameter_info = { + "name": provider_params_field.name, + "type": str(provider_params_field.type).replace("typing.", ""), + "description": description, + "default": default, + "optional": True, + "standard": True, + } + + return provider_parameter_info + + +def get_provider_field_params( + model_map: Dict[str, Any], + params_type: str, + provider: str = "openbb", +) -> List[Dict[str, Any]]: + """Get the fields of the given parameter type for the given provider + of the standard_model. + + Args: + provider_map (Dict[str, Any]): Model Map containing the QueryParams and Data parameters + params_type (str): Parameters to fetch data for (QueryParams or Data) + provider (str, optional): Provider name. Defaults to "openbb". + + Returns: + List[Dict[str, str]]: List of dictionaries containing the field name, + type, description, default, optional flag and standard flag for each provider. + """ + + provider_field_params = [] + expanded_types = MethodDefinition.TYPE_EXPANSION + + for field, field_info in model_map[provider][params_type]["fields"].items(): + # Determine the field type, expanding it if necessary and if params_type is "Parameters" + field_type = get_field_data_type(field_info.annotation) + + if params_type == "QueryParams" and field in expanded_types: + expanded_type = get_field_data_type(expanded_types[field]) + field_type = f"Union[{expanded_type}, {field_type}]" + + cleaned_description = ( + str(field_info.description) + .strip().replace("\n", " ").replace(" ", " ").replace('"', "'") + ) # fmt: skip + + # Add information for the providers supporting multiple symbols + if params_type == "QueryParams" and field_info.json_schema_extra: + multiple_items = ", ".join( + field_info.json_schema_extra["multiple_items_allowed"] + ) + cleaned_description += ( + f" Multiple items allowed for provider(s): {multiple_items}." + ) + # Manually setting to List[<field_type>] for multiple items + # Should be removed if TYPE_EXPANSION is updated to include this + field_type = f"Union[{field_type}, List[{field_type}]]" + + default_value = "" if field_info.default is PydanticUndefined else str(field_info.default) # fmt: skip + + provider_field_params.append( { - "name": param.arg_name, - "type": get_annotation_type(param.type_name), - "default": ( - str(arg_default) - if arg_default is not inspect.Parameter.empty - else None - ), - "cleaned_type": re.sub( - r"Literal\[([^\"\]]*)\]", - f"Literal[{type(arg_default).__name__}]", - get_annotation_type( - formatted_params[param.arg_name].annotation - if param.arg_name in formatted_params - else param.type_name - ), - ), - "optional": bool(arg_default is not inspect.Parameter.empty) - or param.is_optional, - "doc": param.description, + "name": field, + "type": field_type, + "description": cleaned_description, + "default": default_value, + "optional": not field_info.is_required(), + "standard": provider == "openbb", } ) - if doc_parsed.returns: - meta_command["returns"] = { - "type": doc_parsed.returns.type_name, - "doc": doc_parsed.returns.description, - } + return provider_field_params + + +def get_function_params_default_value(endpoint: Callable) -> Dict: + """Get the default for the endpoint function parameters. + + Args: + endpoint (Callable): Router endpoint function + + Returns: + Dict: Endpoint function parameters and their default values + """ + + default_values = {} + + signature = inspect.signature(endpoint) + parameters = signature.parameters + + for name, param in parameters.items(): + if param.default is not inspect.Parameter.empty: + default_values[name] = param.default + else: + default_values[name] = "" + + return default_values + + +def get_post_method_parameters_info(endpoint: Callable) -> List[Dict[str, str]]: + """Get the parameters for the POST method endpoints. + + Args: + endpoint (Callable): Router endpoint function + + Returns: + List[Dict[str, str]]: List of dictionaries containing the name, + type, description, default and optionality of each parameter. + """ + parameters_info = [] + descriptions = {} + + parameters_default_values = get_function_params_default_value(endpoint) + section = endpoint.__doc__.split("Parameters")[1].split("Returns")[0] # type: ignore - examples = [] - for example in doc_parsed.examples: - examples.append( + lines = section.split("\n") + current_param = None + for line in lines: + cleaned_line = line.strip() + + if ":" in cleaned_line: # This line names a parameter + current_param = cleaned_line.split(":")[0] + current_param = current_param.strip() + elif current_param: # This line describes the parameter + description = cleaned_line.strip() + descriptions[current_param] = description + # Reset current_param to ensure each description is + # correctly associated with the parameter + current_param = None + + for param, param_type in endpoint.__annotations__.items(): + if param == "return": + continue + + parameters_info.append( { - "snippet": example.snippet, - "description": example.description.strip(), # type: ignore + "name": param, + "type": get_field_data_type(param_type), + "description": descriptions.get(param, ""), + "default": parameters_default_values.get(param, ""), + "optional": "Optional" in str(param_type), } ) - def_params = [ - f"{d['name']}: {d['cleaned_type']}{' = ' + d['default'] if d['default'] else ''}" - for d in cmd_params - ] - meta_command.update( + return parameters_info + + +def get_post_method_returns_info(endpoint: Callable) -> List[Dict[str, str]]: + """Get the returns information for the POST method endpoints. + + Args: + endpoint (Callable): Router endpoint function + + Returns: + Dict[str, str]: Dictionary containing the name, type, description of the return value + """ + section = endpoint.__doc__.split("Parameters")[1].split("Returns")[-1] # type: ignore + description_lines = section.strip().split("\n") + description = description_lines[-1].strip() if len(description_lines) > 1 else "" + return_type = endpoint.__annotations__["return"].model_fields["results"].annotation + + # Only one item is returned hence its a list with a single dictionary. + # Future changes to the return type will require changes to this code snippet. + return_info = [ { - "description": doc_parsed.short_description - + ( - "\n\n" + doc_parsed.long_description - if doc_parsed.long_description - else "" - ), - "params": cmd_params, - "func_def": f"{full_command_path}({', '.join(def_params)})", - "examples": examples, + "name": "results", + "type": get_field_data_type(return_type), + "description": description, } - ) + ] - return meta_command + return return_info -def generate_markdown(meta_command: dict): - markdown = meta_command["header"] +# mypy: disable-error-code="attr-defined,arg-type" +def generate_reference_file() -> None: + """Generate reference.json file using the ProviderInterface map.""" - markdown += "<!-- markdownlint-disable MD012 MD031 MD033 -->\n\n" - markdown += ( - "import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n" - ) + # ProviderInterface Map contains the model and its + # corresponding QueryParams and Data fields + pi_map = ProviderInterface().map + reference: Dict[str, Dict] = {} - markdown += generate_markdown_section(meta_command) - return markdown + # Fields for the reference dictionary to be used in the JSON file + REFERENCE_FIELDS = [ + "deprecated", + "description", + "examples", + "parameters", + "returns", + "data", + ] + # Router object is used to get the endpoints and their + # corresponding APIRouter object + router = RouterLoader.from_extensions() + route_map = {route.path: route for route in router.api_router.routes} -def generate_markdown_section(meta: Dict[str, Any]): - # Process description to handle docstring examples - lines = meta["description"].split("\n") - description = [] - example_code = [] - in_example_block = False + for path, route in route_map.items(): + # Initialize the reference fields as empty dictionaries + reference[path] = {field: {} for field in REFERENCE_FIELDS} - for line in lines: - if line.strip().startswith(">>>"): - in_example_block = True - # Remove leading '>>>' and spaces - example_line = line.strip()[4:] - example_code.append(example_line) - else: - if in_example_block: - # We've reached the end of an example block - in_example_block = False - # Append the gathered example code as a block - description.append("```python\n" + "\n".join(example_code) + "\n```\n") - example_code = [] # Reset for the next example block - # Add the current line to the description - description.append(line.strip()) - - prev_snippet = " " - for example in meta.get("examples", []): - if isinstance(example["snippet"], str) and ">>>" in example["snippet"]: - snippet = example["snippet"].replace(">>> ", "") - example_code.append(snippet) - if example["description"] and prev_snippet != "": - example_code.append(example["description"]) - prev_snippet = snippet.strip() - elif example["description"]: - example_code.append(example["description"]) - else: - if example["description"]: - example_code.append(example["description"]) - prev_snippet = "" + # Route method is used to distinguish between GET and POST methods + route_method = route.methods - # Join the description parts and handle any remaining example code - if example_code: # If there's an example block at the end of the docstring - if meta.get("examples", []): - description.append("\nExample:\n-------\n") - description.append("\n\n```python\n" + "\n".join(example_code) + "\n```") + # Route endpoint is the callable function + route_func = route.endpoint - markdown_description = "\n".join(description) + # Standard model is used as the key for the ProviderInterface Map dictionary + standard_model = route.openapi_extra["model"] if route_method == {"GET"} else "" - markdown = markdown_description - markdown += "\n\n" if not markdown_description.endswith("\n\n") else "" + # Model Map contains the QueryParams and Data fields for each provider for a standard model + model_map = pi_map[standard_model] if standard_model else "" - # Only add function definition if there was no example code - if not example_code and not re.search(r"```python", markdown): - markdown += "```python wordwrap\n" + meta["func_def"] + "\n```\n\n" + # Add endpoint model for GET methods + reference[path]["model"] = standard_model - markdown += "---\n\n## Parameters\n\n" - if meta["params"]: - markdown += generate_params_markdown_section(meta) - else: - markdown += "This function does not take standardized parameters.\n\n" + # Add endpoint deprecation details + deprecated_value = getattr(route, "deprecated", None) + reference[path]["deprecated"] = { + "flag": bool(deprecated_value), + "message": route.summary if deprecated_value else None, + } - markdown += "---\n\n## Returns\n\n" - if meta["returns"]: - return_desc = meta["returns"]["doc"] if meta["returns"]["doc"] else "" - markdown += f"```python wordwrap\n{return_desc}\n```\n\n" - else: - markdown += "This function does not return a standardized model\n\n" + # Add endpoint description + if route_method == {"GET"}: + reference[path]["description"] = route.description + elif route_method == {"POST"}: + # POST method router `description` attribute is unreliable as it may or + # may not contain the "Parameters" and "Returns" sections. Hence, the + # endpoint function docstring is used instead. + description = route.endpoint.__doc__.split("Parameters")[0].strip() + # Remove extra spaces in between the string + reference[path]["description"] = re.sub(" +", " ", description) + + # Add endpoint examples + reference[path]["examples"] = route.openapi_extra.get("examples", []) + + # Add endpoint parameters fields for standard provider + if route_method == {"GET"}: + # openbb provider is always present hence its the standard field + reference[path]["parameters"]["standard"] = get_provider_field_params( + model_map, "QueryParams" |