path: root/integrations/gen_docs_integrations.py

import json
import shutil
from pathlib import Path
import re

# Dictionary responsible for making the symbolic links at the end of the script's run.
symlink_dict = {}


def cleanup():
    """
    clean directories that are either data collection or exporting integrations
    """
    for element in Path("src/go/collectors/go.d.plugin/modules").glob('**/*/'):
        if "integrations" in str(element):
            shutil.rmtree(element)
    for element in Path("src/collectors").glob('**/*/'):
        # print(element)
        if "integrations" in str(element):
            shutil.rmtree(element)

    for element in Path("src/exporting").glob('**/*/'):
        if "integrations" in str(element):
            shutil.rmtree(element)
    for element in Path("integrations/cloud-notifications").glob('**/*/'):
        if "integrations" in str(element) and not "metadata.yaml" in str(element):
            shutil.rmtree(element)
    for element in Path("integrations/cloud-authentication").glob('**/*/'):
        if "integrations" in str(element) and not "metadata.yaml" in str(element):
            shutil.rmtree(element)

def generate_category_from_name(category_fragment, category_array):
    """
    Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
    """

    category_name = ""
    i = 0
    dummy_id = category_fragment[0]

    while i < len(category_fragment):
        for category in category_array:

            if dummy_id == category['id']:
                category_name = category_name + "/" + category["name"]
                try:
                    # print("equals")
                    # print(fragment, category_fragment[i+1])
                    dummy_id = dummy_id + "." + category_fragment[i+1]
                    # print(dummy_id)
                except IndexError:
                    return category_name.split("/", 1)[1]
                category_array = category['children']
                break
        i += 1


def clean_and_write(md, path):
    """
    This function takes care of the special details element, and converts it to the equivalent that md expects.
    Then it writes the buffer on the file provided.
    """
    # clean first, replace
    md = md.replace("{% details summary=\"", "<details><summary>").replace(
        "\" %}", "</summary>\n").replace("{% /details %}", "</details>\n")

    path.write_text(md)


def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'):
    """
    Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field
    """

    output = ""

    if mode == 'default':
        path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'

    elif mode == 'cloud-notifications':
        path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')

    elif mode == 'agent-notifications':
        path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")

    elif mode == 'cloud-authentication':
        path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')

    output = markdown_string.replace(
        "<!--startmeta",
        f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"')

    return output


def clean_string(string):
    """
    simple function to get rid of caps, spaces, slashes and parentheses from a given string

    The string represents an integration name, as it would be displayed in the final text
    """

    return string.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "").replace(":", "")


def read_integrations_js(path_to_file):
    """
    Open integrations/integrations.js and extract the dictionaries
    """

    try:
        data = Path(path_to_file).read_text()

        categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
        integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]

        return json.loads(categories_str), json.loads(integrations_str)

    except FileNotFoundError as e:
        print("Exception", e)


def create_overview(integration, filename):

    split = re.split(r'(#.*\n)', integration['overview'], 1)

    first_overview_part = split[1]
    rest_overview_part = split[2]

    if len(filename) > 0:
        return f"""{first_overview_part}

<img src="https://netdata.cloud/img/{filename}" width="150"/>

{rest_overview_part}
"""
    else:
        return f"""{first_overview_part}{rest_overview_part}
"""


def build_readme_from_integration(integration, mode=''):
    # COLLECTORS
    if mode == 'collector':

        try:
            # initiate the variables for the collector
            meta_yaml = integration['edit_link'].replace("blob", "edit")
            sidebar_label = integration['meta']['monitored_instance']['name']
            learn_rel_path = generate_category_from_name(
                integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace("Data Collection", "Collecting Metrics")
            most_popular = integration['meta']['most_popular']

            # build the markdown string
            md = \
                f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path}"
most_popular: {most_popular}
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE"
endmeta-->

{create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}"""

            if integration['metrics']:
                md += f"""
{integration['metrics']}
"""

            if integration['alerts']:
                md += f"""
{integration['alerts']}
"""

            if integration['setup']:
                md += f"""
{integration['setup']}
"""

            if integration['troubleshooting']:
                md += f"""
{integration['troubleshooting']}
"""
        except Exception as e:
            print("Exception in collector md construction", e, integration['id'])

    # EXPORTERS
    elif mode == 'exporter':
        try:
            # initiate the variables for the exporter
            meta_yaml = integration['edit_link'].replace("blob", "edit")
            sidebar_label = integration['meta']['name']
            learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)

            # build the markdown string
            md = \
                f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "Exporting Metrics"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE EXPORTER'S metadata.yaml FILE"
endmeta-->

{create_overview(integration, integration['meta']['icon_filename'])}"""

            if integration['setup']:
                md += f"""
{integration['setup']}
"""

            if integration['troubleshooting']:
                md += f"""
{integration['troubleshooting']}
"""
        except Exception as e:
            print("Exception in exporter md construction", e, integration['id'])

    # NOTIFICATIONS
    elif mode == 'notification':
        try:
            # initiate the variables for the notification method
            meta_yaml = integration['edit_link'].replace("blob", "edit")
            sidebar_label = integration['meta']['name']
            learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)

            # build the markdown string
            md = \
                f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}"
message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
endmeta-->

{create_overview(integration, integration['meta']['icon_filename'])}"""

            if integration['setup']:
                md += f"""
{integration['setup']}
"""

            if integration['troubleshooting']:
                md += f"""
{integration['troubleshooting']}
"""

        except Exception as e:
            print("Exception in notification md construction", e, integration['id'])
    
    # AUTHENTICATIONS
    elif mode == 'authentication':
        if True:
            # initiate the variables for the authentication method
            meta_yaml = integration['edit_link'].replace("blob", "edit")
            sidebar_label = integration['meta']['name']
            learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)

            # build the markdown string
            md = \
                f"""<!--startmeta
meta_yaml: "{meta_yaml}"
sidebar_label: "{sidebar_label}"
learn_status: "Published"
learn_rel_path: "{learn_rel_path.replace("authentication", "Authentication")}"