Test for provider field duplication check (#5433)

* bump pytest and add pytest-subtests

* add the test for dupes verification

* ruff

* Small cleanup

* Revert "Small cleanup"

This reverts commit 4e935aafac65de948b9b7e193455a2f2ed8e72f3.

* getting @IgorWounds adjustments back withouth the static assets

* resolving @the-praxs suggestions

* better error messages and provider:field matching for better debugging

* Test providers + cov

* ignore nested test folders in codecov

* install all

* james wrong copy-paste

* adding `-e` flag

* Revert "adding `-e` flag"

This reverts commit c0c33709a68cae63c318ec85cc1045079e2fec4c.

* correct wildcard

* changing the way we obtain provider modules

* adding init to test in the ci?

* omitting package auto generated files

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
Co-authored-by: James Maslek <jmaslek11@gmail.com>
Co-authored-by: Pratyush Shukla <ps4534@nyu.edu>

8 files changed, 275 insertions, 52 deletions

diff --git a/.coveragerc b/.coveragerc
index 81e405bf455..081cc71dbc6 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -2,4 +2,6 @@
 omit =
     env/*
     tests/*
+    **/tests/**
+    **/package/**
 source = .
diff --git a/noxfile.py b/noxfile.py
index 9f92d0e0f6e..1344df3d631 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,8 +1,11 @@
 import nox
 
+test_locations = ["openbb_sdk/sdk/core/tests", "openbb_sdk/providers/tests"]
+
 
 @nox.session(python=["3.8", "3.9", "3.10", "3.11"])
 def tests(session):
-    session.install("./openbb_sdk")
+    session.install("./openbb_sdk[all]")
     session.install("pytest")
-    session.run("pytest", "openbb_sdk/sdk/core/tests")
+    session.install("pytest-cov")
+    session.run("pytest", *test_locations, "--cov=openbb_sdk/")
diff --git a/openbb_sdk/poetry.lock b/openbb_sdk/poetry.lock
index 076549bebbd..911e38e40c8 100644
--- a/openbb_sdk/poetry.lock
+++ b/openbb_sdk/poetry.lock
@@ -216,16 +216,6 @@ files = [
 ]
 
 [[package]]
-name = "atomicwrites"
-version = "1.4.1"
-description = "Atomic file writes."
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-files = [
-    {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"},
-]
-
-[[package]]
 name = "attrs"
 version = "23.1.0"
 description = "Classes Without Boilerplate"
@@ -308,13 +298,33 @@ lxml = ["lxml"]
 
 [[package]]
 name = "black"
-version = "23.9.0"
+version = "23.9.1"
 description = "The uncompromising code formatter."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "black-23.9.0-py3-none-any.whl", hash = "sha256:9366c1f898981f09eb8da076716c02fd021f5a0e63581c66501d68a2e4eab844"},
-    {file = "black-23.9.0.tar.gz", hash = "sha256:3511c8a7e22ce653f89ae90dfddaf94f3bb7e2587a245246572d3b9c92adf066"},
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"},
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"},
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"},
+    {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"},
+    {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"},
+    {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"},
+    {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"},
+    {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"},
+    {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"},
+    {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"},
+    {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"},
+    {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"},
+    {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"},
 ]
 
 [package.dependencies]
@@ -1992,17 +2002,6 @@ files = [
 ]
 
 [[package]]
-name = "py"
-version = "1.11.0"
-description = "library with cross-python path, ini-parsing, io, code, log facilities"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-files = [
-    {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
-    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
-]
-
-[[package]]
 name = "pyasn1"
 version = "0.5.0"
 description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
@@ -2087,27 +2086,40 @@ tests = ["pytest", "pytest-xdist"]
 
 [[package]]
 name = "pytest"
-version = "6.2.5"
+version = "7.4.2"
 description = "pytest: simple powerful testing with Python"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"
 files = [
-    {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
-    {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
+    {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"},
+    {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"},
 ]
 
 [package.dependencies]
-atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
-attrs = ">=19.2.0"
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
 pluggy = ">=0.12,<2.0"
-py = ">=1.8.2"
-toml = "*"
+tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
 
 [package.extras]
-testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
+testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
+[[package]]
+name = "pytest-subtests"
+version = "0.11.0"
+description = "unittest subTest() support and subtests fixture"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-subtests-0.11.0.tar.gz", hash = "sha256:51865c88457545f51fb72011942f0a3c6901ee9e24cbfb6d1b9dc1348bafbe37"},
+    {file = "pytest_subtests-0.11.0-py3-none-any.whl", hash = "sha256:453389984952eec85ab0ce0c4f026337153df79587048271c7fd0f49119c07e4"},
+]
+
+[package.dependencies]
+attrs = ">=19.2.0"
+pytest = ">=7.0"
 
 [[package]]
 name = "python-dateutil"
@@ -2651,19 +2663,19 @@ test = ["pytest"]
 
 [[package]]
 name = "setuptools"
-version = "68.2.0"
+version = "68.2.1"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 optional = true
 python-versions = ">=3.8"
 files = [
-    {file = "setuptools-68.2.0-py3-none-any.whl", hash = "sha256:af3d5949030c3f493f550876b2fd1dd5ec66689c4ee5d5344f009746f71fd5a8"},
-    {file = "setuptools-68.2.0.tar.gz", hash = "sha256:00478ca80aeebeecb2f288d3206b0de568df5cd2b8fada1209843cc9a8d88a48"},
+    {file = "setuptools-68.2.1-py3-none-any.whl", hash = "sha256:eff96148eb336377ab11beee0c73ed84f1709a40c0b870298b0d058828761bae"},
+    {file = "setuptools-68.2.1.tar.gz", hash = "sha256:56ee14884fd8d0cd015411f4a13f40b4356775a0aefd9ebc1d3bfb9a1acb32f1"},
 ]
 
 [package.extras]
 docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
 testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
-testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
 
 [[package]]
 name = "setuptools-scm"
@@ -2845,17 +2857,6 @@ doc = ["sphinx", "sphinx_rtd_theme"]
 test = ["flake8", "isort", "pytest"]
 
 [[package]]
-name = "toml"
-version = "0.10.2"
-description = "Python Library for Tom's Obvious, Minimal Language"
-optional = false
-python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
-files = [
-    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
-    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
-]
-
-[[package]]
 name = "tomli"
 version = "2.0.1"
 description = "A lil' TOML parser"
@@ -3275,4 +3276,4 @@ yfinance = ["openbb-yfinance"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.12"
-content-hash = "468558e372f483a87864db0d5220e56937b8b2b1c8ad67a614d959d62ccb19e4"
+content-hash = "b88ccaa7809833588bdae056576238ede6670f0c8ce43126d81c4757bbd3f30c"
diff --git a/openbb_sdk/providers/__init__.py b/openbb_sdk/providers/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/openbb_sdk/providers/__init__.py
diff --git a/openbb_sdk/providers/tests/.gitkeep b/openbb_sdk/providers/tests/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/openbb_sdk/providers/tests/.gitkeep
diff --git a/openbb_sdk/providers/tests/__init__.py b/openbb_sdk/providers/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/openbb_sdk/providers/tests/__init__.py
diff --git a/openbb_sdk/providers/tests/test_provider_field_dupes.py b/openbb_sdk/providers/tests/test_provider_field_dupes.py
new file mode 100644
index 00000000000..6bf4efff95e
--- /dev/null
+++ b/openbb_sdk/providers/tests/test_provider_field_dupes.py
@@ -0,0 +1,216 @@
+import glob
+import importlib
+import inspect
+import os
+import unittest
+from typing import Dict, List, Type
+
+from openbb_provider import standard_models
+from openbb_provider.abstract.data import Data
+from openbb_provider.abstract.query_params import QueryParams
+from openbb_provider.registry import RegistryLoader
+
+
+def get_module(file_path: str, package_name: str):
+    """
+    Given a file path and its package, loads the module.
+
+    Parameters
+    ----------
+        file_path (str): The path to the file.
+        package (str): The package where the module is located.
+
+    Returns
+    -------
+        module: The loaded module.
+    """
+    # Get the module name by removing the file extension and getting the basename
+    module_name = os.path.splitext(os.path.basename(file_path))[0]
+
+    # Import the module using the package name and the module name
+    return importlib.import_module(f"{package_name}.{module_name}")
+
+
+def get_subclasses_w_keys(module: object, cls: Type) -> Dict[Type, List[str]]:
+    """Given a module and a class, return the subclasses of the class and their fields.
+
+    Parameters
+    ----------
+        module (object): The module containing the classes.
+        cls (Type): The base class.
+
+    Returns
+    -------
+        Dict[Type, List[str]]: A dictionary mapping each subclass to a list of its field names.
+    """
+    subclasses = {}
+    module_members = inspect.getmembers(module)
+
+    for _, obj in module_members:
+        if inspect.isclass(obj) and issubclass(obj, cls) and obj != cls:
+            subclasses[obj] = list(obj.__fields__.keys())
+    return subclasses
+
+
+def get_subclasses(
+    python_files: List[str], package_name: str, cls: Type
+) -> Dict[Type, List[str]]:
+    """
+    Given a list of python files, and a class, return a dictionary of
+    subclasses of that class that are defined in those files.
+
+    Parameters
+    ----------
+        python_files (List[str]): A list of file paths to Python files.
+        package_name (str): The name of the package.
+        cls (Type): The base class.
+
+    Returns
+    -------
+        Dict[str, Type]: A dictionary where the keys are the subclass names
+        and the values are the subclasses themselves.
+    """
+    subclasses = {}
+
+    for file_path in python_files:
+        module = get_module(file_path, package_name)
+
+        subclasses.update(get_subclasses_w_keys(module, cls))
+
+    return subclasses
+
+
+def child_parent_map(map_: Dict, parents: Dict, module: object) -> None:
+    """
+    Generate a mapping of child classes to their parent classes and provider fields.
+
+    Parameters
+    ----------
+        map_ (dict): The dict to append to.
+        parents (dict): A dictionary of parent classes and their standard fields.
+        module (module): The module containing the classes.
+    """
+    for cls, std_fields in parents.items():
+        # Check if class name is not already in the map
+        if cls.__name__ not in map_:
+            map_[cls.__name__] = []
+
+        # Get the first subclass and its provider fields
+        sub_w_keys = get_subclasses_w_keys(module, cls)
+        if sub_w_keys:
+            subclass = list(sub_w_keys.keys())[0]
+            provider_fields = list(sub_w_keys.values())[0]
+
+            # Remove standard fields from provider fields
+            provider_fields = [
+                field for field in provider_fields if field not in std_fields
+            ]
+
+            # If there are provider fields, add them to the map
+            if provider_fields:
+                map_[cls.__name__].append({subclass.__name__: provider_fields})
+
+
+def get_path_components(path: str):
+    """Given a path, return a list of path components"""
+
+    path_components = []
+    head, tail = os.path.split(path)
+
+    while tail:
+        path_components.append(tail)
+        head, tail = os.path.split(head)
+
+    return path_components
+
+
+def match_provider_and_fields(
+    providers_w_fields: List[Dict[str, List[str]]], duplicated_fields: List[str]
+) -> List[str]:
+    """
+    Given a list of providers with fields and duplicated fields,
+    return a list of matching "Provider:'dup_field'".
+    """
+    matching_provider_fields = []
+
+    for item in providers_w_fields:
+        for model, fields in item.items():
+            for f in duplicated_fields:
+                if f in fields:
+                    matching_provider_fields.append(f"{model}:'{f}'")
+
+    return matching_provider_fields
+
+
+def get_provider_modules():
+    """Get provider modules."""
+    registry = RegistryLoader.from_extensions()
+    modules = []
+    for _, provider in registry.providers.items():
+        for _, fetcher in provider.fetcher_dict.items():
+            modules.append(fetcher.__module__)
+    return modules
+
+
+class ProviderFieldDupesTest(unittest.TestCase):
+    """Test for common fields in the provider models that should be standard."""
+
+    def test_provider_field_dupes(self):
+        """
+        This function checks for duplicate fields in the provider models
+        and identifies the fields that should be standardized.
+        """
+
+        standard_models_directory = os.path.dirname(standard_models.__file__)
+        standard_models_files = glob.glob(
+            os.path.join(standard_models_directory, "*.py")
+        )
+
+        standard_query_classes = get_subclasses(
+            standard_models_files, standard_models.__name__, QueryParams
+        )
+        standard_data_classes = get_subclasses(
+            standard_models_files, standard_models.__name__, Data
+        )
+
+        provider_modules = get_provider_modules()
+
+        child_parent_dict = {}
+
+        for module in provider_modules:
+            provider_module = importlib.import_module(module)
+
+            # query classes
+            child_parent_map(child_parent_dict, standard_query_classes, provider_module)
+
+            # data classes
+            child_parent_map(child_parent_dict, standard_data_classes, provider_module)
+
+        # remove keys with no values
+        child_parent_dict = {k: v for k, v in child_parent_dict.items() if v}
+
+        for std_cls in child_parent_dict:
+            with self.subTest(i=std_cls):
+                providers_w_fields = child_parent_dict[std_cls]
+
+                fields = []
+                provider_models = []
+
+                for provider_cls in providers_w_fields:
+                    provider_models.extend(list(provider_cls.keys()))
+                    fields.extend(list(provider_cls.values())[0])
+
+                seen = set()
+                dupes = [x for x in fields if x in seen or seen.add(x)]
+
+                dupes_str = ", ".join([f"'{x}'" for x in set(dupes)])
+                provider_str = (
+                    ", ".join(match_provider_and_fields(providers_w_fields, set(dupes)))
+                    if dupes
+                    else ""
+                )
+
+                assert not dupes, (
+                    f"The following fields are common among models and should be standardized: {dupes_str}.\n"
+                    f"Standard model: {std_cls}, Provider models: {provider_str}\n"
+                )
diff --git a/openbb_sdk/pyproject.toml b/openbb_sdk/pyproject.toml
index aac1962a0a3..70bb557ba20 100644
--- a/openbb_sdk/pyproject.toml
+++ b/openbb_sdk/pyproject.toml
@@ -59,7 +59,8 @@ all = [
 ]
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^6.2.2"
+pytest = "^7.0.0"
+pytest-subtests = "^0.11.0"
 
 [build-system]
 requires = ["poetry-core"]