diff options
author | Austin S. Hemmelgarn <austin@netdata.cloud> | 2023-08-01 17:23:16 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-08-01 17:23:16 -0400 |
commit | 183bb1db1903d2742e16618af2cd63a7c6eddb28 (patch) | |
tree | 6f54fad2c3d2fde003312549046f08374f740ef0 /integrations/gen_integrations.py | |
parent | 7773b5ee330014dced7b037d0774e41dd68efafa (diff) |
Add initial tooling for generating integrations.js file. (#15406)
* Fix link tags in deploy.
* Add initial tooling for generating integrations.js file.
* Skip integrations directory for eslint.
* Add README to explain how to generate integrations.js locally.
* Fix ID/name for top-level categories.
* Deduplicate categories entries.
* Properly render related resources information.
* Warn on and skip bad references for related resources.
* Add CI workflow to rebuild integrations as-needed.
* Add integrations.js to build artifacts.
* Fix actionlint complaints.
* Assorted template fixes.
* Add script to check collector metadata.
* Add default categories for collectors when they have no categories.
* Fix template formatting issues.
* Link related resources properly.
* Skip more sections in rendered output if they are not present in source data.
* Temporarily skip config syntax section.
It needs further work and is not critical at the moment.
* Fix metrics table rendering.
* Hide most overview content if method_description is empty.
* Fix metrics table rendering (again).
* Add detailed description to setup options section.
* Fix detailed description handling for config options.
* Fix config example folding logic.
* Fix multi-instance selection.
* Properly fix multi-instance selection.
* Add titles for labels and metrics charts.
* Include monitored instance name in integration ID.
This is required to disambiguate some ‘virtual’ integrations.
* Indicate if there are no alerts defined for an integration.
* Fix multi-instance in template.
* Improve warning handling in script and fix category handling.
* Hide debug messages by default.
* Fix invalid category name in cgroups plugin.
* Completely fix invalid categories in cgroups plugin.
* Warn about and ignore duplicate integration ids.
* Flag integration type in integrations list.
* Add configuration syntax samples.
* Fix issues in gen_integrations.py
* Validate categories.yaml on load.
* Add support for handling deployment information.
* Fix bugs in gen_integrations.py
* Add code to handle exporters.
* Add link to integrations pointing to their source files.
* Fix table justification.
* Add notification handling to script.
Also tidy up a few other things.
* Fix numerous bugs in gen_integrations.py
* remove trailing space from deploy.yaml command
* make availability one column
* Switch back to multiple columns for availability.
And also switch form +/- to a dot for positive and empty cell for
negative.
* Render setup description.
* Fix platform info rendering in deploy integrations.
* Fix sourcing of cloud-notifications metadata.
* Fix rendering of empty metrics.
* Fix alerts template.
* Add per-instance templating for templated keys.
* Fix go plugin links.
* Fix overview template.
* Fix handling of exporters.
* Fix loading of cloud notification integrations.
* Always show full collector overview.
* Add static troubleshooting content when appropriate.
* Assorted deploy integration updates.
* Add initial copy of integrations.js.
---------
Co-authored-by: Fotis Voutsas <fotis@netdata.cloud>
Diffstat (limited to 'integrations/gen_integrations.py')
-rwxr-xr-x | integrations/gen_integrations.py | 617 |
1 files changed, 617 insertions, 0 deletions
diff --git a/integrations/gen_integrations.py b/integrations/gen_integrations.py new file mode 100755 index 0000000000..0246561ceb --- /dev/null +++ b/integrations/gen_integrations.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 + +import json +import os +import sys + +from pathlib import Path + +from jsonschema import Draft7Validator, ValidationError +from referencing import Registry, Resource +from referencing.jsonschema import DRAFT7 +from ruamel.yaml import YAML, YAMLError + +AGENT_REPO = 'netdata/netdata' +GO_REPO = 'netdata/go.d.plugin' + +INTEGRATIONS_PATH = Path(__file__).parent +TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates' +OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js' +CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml' +REPO_PATH = INTEGRATIONS_PATH.parent +SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas' +GO_REPO_PATH = REPO_PATH / 'go.d.plugin' +DISTROS_FILE = REPO_PATH / '.github' / 'data' / 'distros.yml' +METADATA_PATTERN = '*/metadata.yaml' + +COLLECTOR_SOURCES = [ + (AGENT_REPO, REPO_PATH / 'collectors', True), + (AGENT_REPO, REPO_PATH / 'collectors' / 'charts.d.plugin', True), + (AGENT_REPO, REPO_PATH / 'collectors' / 'python.d.plugin', True), + (GO_REPO, GO_REPO_PATH / 'modules', True), +] + +DEPLOY_SOURCES = [ + (AGENT_REPO, INTEGRATIONS_PATH / 'deploy.yaml', False), +] + +EXPORTER_SOURCES = [ + (AGENT_REPO, REPO_PATH / 'exporting', True), +] + +NOTIFICATION_SOURCES = [ + (AGENT_REPO, REPO_PATH / 'health' / 'notifications', True), + (AGENT_REPO, INTEGRATIONS_PATH / 'cloud-notifications' / 'metadata.yaml', False), +] + +COLLECTOR_RENDER_KEYS = [ + 'alerts', + 'metrics', + 'overview', + 'related_resources', + 'setup', + 'troubleshooting', +] + +EXPORTER_RENDER_KEYS = [ + 'overview', + 'setup', +] + +NOTIFICATION_RENDER_KEYS = [ + 'overview', + 'setup', +] + +GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False) +DEBUG = os.environ.get('DEBUG', False) + + +def debug(msg): + if GITHUB_ACTIONS: + print(f':debug:{ msg }') + elif DEBUG: + print(f'>>> { msg }') + else: + pass + + +def warn(msg, path): + if GITHUB_ACTIONS: + print(f':warning file={ path }:{ msg }') + else: + print(f'!!! WARNING:{ path }:{ msg }') + + +def retrieve_from_filesystem(uri): + path = SCHEMA_PATH / Path(uri) + contents = json.loads(path.read_text()) + return Resource.from_contents(contents, DRAFT7) + + +registry = Registry(retrieve=retrieve_from_filesystem) + +CATEGORY_VALIDATOR = Draft7Validator( + {'$ref': './categories.json#'}, + registry=registry, +) + +DEPLOY_VALIDATOR = Draft7Validator( + {'$ref': './deploy.json#'}, + registry=registry, +) + +EXPORTER_VALIDATOR = Draft7Validator( + {'$ref': './exporter.json#'}, + registry=registry, +) + +NOTIFICATION_VALIDATOR = Draft7Validator( + {'$ref': './notification.json#'}, + registry=registry, +) + +COLLECTOR_VALIDATOR = Draft7Validator( + {'$ref': './collector.json#'}, + registry=registry, +) + +_jinja_env = False + + +def get_jinja_env(): + global _jinja_env + + if not _jinja_env: + from jinja2 import Environment, FileSystemLoader, select_autoescape + + _jinja_env = Environment( + loader=FileSystemLoader(TEMPLATE_PATH), + autoescape=select_autoescape(), + block_start_string='[%', + block_end_string='%]', + variable_start_string='[[', + variable_end_string=']]', + comment_start_string='[#', + comment_end_string='#]', + trim_blocks=True, + lstrip_blocks=True, + ) + + return _jinja_env + + +def get_category_sets(categories): + default = set() + valid = set() + + for c in categories: + if 'id' in c: + valid.add(c['id']) + + if c.get('collector_default', False): + default.add(c['id']) + + if 'children' in c and c['children']: + d, v = get_category_sets(c['children']) + default |= d + valid |= v + + return (default, valid) + + +def get_collector_metadata_entries(): + ret = [] + + for r, d, m in COLLECTOR_SOURCES: + if d.exists() and d.is_dir() and m: + for item in d.glob(METADATA_PATTERN): + ret.append((r, item)) + elif d.exists() and d.is_file() and not m: + if d.match(METADATA_PATTERN): + ret.append(d) + + return ret + + +def load_yaml(src): + yaml = YAML(typ='safe') + + if not src.is_file(): + warn(f'{ src } is not a file.', src) + return False + + try: + contents = src.read_text() + except (IOError, OSError): + warn(f'Failed to read { src }.', src) + return False + + try: + data = yaml.load(contents) + except YAMLError: + warn(f'Failed to parse { src } as YAML.', src) + return False + + return data + + +def load_categories(): + categories = load_yaml(CATEGORIES_FILE) + + if not categories: + sys.exit(1) + + try: + CATEGORY_VALIDATOR.validate(categories) + except ValidationError: + warn(f'Failed to validate { CATEGORIES_FILE } against the schema.', CATEGORIES_FILE) + sys.exit(1) + + return categories + + +def load_collectors(): + ret = [] + + entries = get_collector_metadata_entries() + + for repo, path in entries: + debug(f'Loading { path }.') + data = load_yaml(path) + + if not data: + continue + + try: + COLLECTOR_VALIDATOR.validate(data) + except ValidationError: + warn(f'Failed to validate { path } against the schema.', path) + continue + + for idx, item in enumerate(data['modules']): + item['meta']['plugin_name'] = data['plugin_name'] + item['integration_type'] = 'collector' + item['_src_path'] = path + item['_repo'] = repo + item['_index'] = idx + ret.append(item) + + return ret + + +def _load_deploy_file(file, repo): + ret = [] + debug(f'Loading { file }.') + data = load_yaml(file) + + if not data: + return [] + + try: + DEPLOY_VALIDATOR.validate(data) + except ValidationError: + warn(f'Failed to validate { file } against the schema.', file) + return [] + + for idx, item in enumerate(data): + item['integration_type'] = 'deploy' + item['_src_path'] = file + item['_repo'] = repo + item['_index'] = idx + ret.append(item) + + return ret + + +def load_deploy(): + ret = [] + + for repo, path, match in DEPLOY_SOURCES: + if match and path.exists() and path.is_dir(): + for file in path.glob(METADATA_PATTERN): + ret.extend(_load_deploy_file(file, repo)) + elif not match and path.exists() and path.is_file(): + ret.extend(_load_deploy_file(path, repo)) + + return ret + + +def _load_exporter_file(file, repo): + debug(f'Loading { file }.') + data = load_yaml(file) + + if not data: + return [] + + try: + EXPORTER_VALIDATOR.validate(data) + except ValidationError: + warn(f'Failed to validate { file } against the schema.', file) + return [] + + if 'id' in data: + data['integration_type'] = 'exporter' + data['_src_path'] = file + data['_repo'] = repo + data['_index'] = 0 + + return [data] + else: + ret = [] + + for idx, item in enumerate(data): + item['integration_type'] = 'exporter' + item['_src_path'] = file + item['_repo'] = repo + item['_index'] = idx + ret.append(item) + + return ret + + +def load_exporters(): + ret = [] + + for repo, path, match in EXPORTER_SOURCES: + if match and path.exists() and path.is_dir(): + for file in path.glob(METADATA_PATTERN): + ret.extend(_load_exporter_file(file, repo)) + elif not match and path.exists() and path.is_file(): + ret.extend(_load_exporter_file(path, repo)) + + return ret + + +def _load_notification_file(file, repo): + debug(f'Loading { file }.') + data = load_yaml(file) + + if not data: + return [] + + try: + NOTIFICATION_VALIDATOR.validate(data) + except ValidationError: + warn(f'Failed to validate { file } against the schema.', file) + return [] + + if 'id' in data: + data['integration_type'] = 'notification' + data['_src_path'] = file + data['_repo'] = repo + data['_index'] = 0 + + return [data] + else: + ret = [] + + for idx, item in enumerate(data): + item['integration_type'] = 'notification' + item['_src_path'] = file + item['_repo'] = repo + item['_index'] = idx + ret.append(item) + + return ret + + +def load_notifications(): + ret = [] + + for repo, path, match in NOTIFICATION_SOURCES: + if match and path.exists() and path.is_dir(): + for file in path.glob(METADATA_PATTERN): + ret.extend(_load_notification_file(file, repo)) + elif not match and path.exists() and path.is_file(): + ret.extend(_load_notification_file(path, repo)) + + return ret + + +def make_id(meta): + if 'monitored_instance' in meta: + instance_name = meta['monitored_instance']['name'].replace(' ', '_') + elif 'instance_name' in meta: + instance_name = meta['instance_name'] + else: + instance_name = '000_unknown' + + return f'{ meta["plugin_name"] }-{ meta["module_name"] }-{ instance_name }' + + +def make_edit_link(item): + if item['_repo'] == 'netdata/go.d.plugin': + item_path = item['_src_path'].relative_to(GO_REPO_PATH) + else: + item_path = item['_src_path'].relative_to(REPO_PATH) + + return f'https://github.com/{ item["_repo"] }/blob/master/{ item_path }' + + +def sort_integrations(integrations): + integrations.sort(key=lambda i: i['_index']) + integrations.sort(key=lambda i: i['_src_path']) + integrations.sort(key=lambda i: i['id']) + + +def dedupe_integrations(integrations, ids): + tmp_integrations = [] + + for i in integrations: + if ids.get(i['id'], False): + first_path, first_index = ids[i['id']] + warn(f'Duplicate integration ID found at { i["_src_path"] } index { i["_index"] } (original definition at { first_path } index { first_index }), ignoring that integration.', i['_src_path']) + else: + tmp_integrations.append(i) + ids[i['id']] = (i['_src_path'], i['_index']) + + return tmp_integrations, ids + + +def render_collectors(categories, collectors, ids): + debug('Computing default categories.') + + default_cats, valid_cats = get_category_sets(categories) + + debug('Generating collector IDs.') + + for item in collectors: + item['id'] = make_id(item['meta']) + + debug('Sorting collectors.') + + sort_integrations(collectors) + + debug('Removing duplicate collectors.') + + collectors, ids = dedupe_integrations(collectors, ids) + + idmap = {i['id']: i for i in collectors} + + for item in collectors: + debug(f'Processing { item["id"] }.') + + related = [] + + for res in item['meta']['related_resources']['integrations']['list']: + res_id = make_id(res) + + if res_id not in idmap.keys(): + warn(f'Could not find related integration { res_id }, ignoring it.', item['_src_path']) + continue + + related.append({ + 'plugin_name': res['plugin_name'], + 'module_name': res['module_name'], + 'id': res_id, + 'name': idmap[res_id]['meta']['monitored_instance']['name'], + 'info': idmap[res_id]['meta']['info_provided_to_referring_integrations'], + }) + + item_cats = set(item['meta']['monitored_instance']['categories']) + bogus_cats = item_cats - valid_cats + actual_cats = item_cats & valid_cats + + if bogus_cats: + warn(f'Ignoring invalid categories: { ", ".join(bogus_cats) }', item["_src_path"]) + + if not item_cats: + item['meta']['monitored_instance']['categories'] = list(default_cats) + warn(f'{ item["id"] } does not list any caregories, adding it to: { default_cats }', item["_src_path"]) + else: + item['meta']['monitored_instance']['categories'] = list(actual_cats) + + for scope in item['metrics']['scopes']: + if scope['name'] == 'global': + scope['name'] = f'{ item["meta"]["monitored_instance"]["name"] } instance' + + for cfg_example in item['setup']['configuration']['examples']['list']: + if 'folding' not in cfg_example: + cfg_example['folding'] = { + 'enabled': item['setup']['configuration']['examples']['folding']['enabled'] + } + + for key in COLLECTOR_RENDER_KEYS: + template = get_jinja_env().get_template(f'{ key }.md') + data = template.render(entry=item, related=related) + + if 'variables' in item['meta']['monitored_instance']: + template = get_jinja_env().from_string(data) + data = template.render(variables=item['meta']['monitored_instance']['variables']) + + item[key] = data + + item['edit_link'] = make_edit_link(item) + + del item['_src_path'] + del item['_repo'] + del item['_index'] + + return collectors, ids + + +def render_deploy(distros, categories, deploy, ids): + debug('Sorting deployments.') + + sort_integrations(deploy) + + debug('Checking deployment ids.') + + deploy, ids = dedupe_integrations(deploy, ids) + + template = get_jinja_env().get_template('platform_info.md') + + for item in deploy: + debug(f'Processing { item["id"] }.') + + if item['platform_info']['group']: + entries = [ + { + 'version': i['version'], + 'support': i['support_type'], + 'arches': i.get('packages', {'arches': []})['arches'], + 'notes': i['notes'], + } for i in distros[item['platform_info']['group']] if i['distro'] == item['platform_info']['distro'] + ] + else: + entries = [] + + data = template.render(entries=entries) + + item['platform_info'] = data + item['edit_link'] = make_edit_link(item) + + del item['_src_path'] + del item['_repo'] + del item['_index'] + + return deploy, ids + + +def render_exporters(categories, exporters, ids): + debug('Sorting exporters.') + + sort_integrations(exporters) + + debug('Checking exporter ids.') + + exporters, ids = dedupe_integrations(exporters, ids) + + for item in exporters: + for key in EXPORTER_RENDER_KEYS: + template = get_jinja_env().get_template(f'{ key }.md') + data = template.render(entry=item) + + if 'variables' in item['meta']: + template = get_jinja_env().from_string(data) + data = template.render(variables=item['meta']['variables']) + + item[key] = data + + item['edit_link'] = make_edit_link(item) + + del item['_src_path'] + del item['_repo'] + del item['_index'] + + return exporters, ids + + +def render_notifications(categories, notifications, ids): + debug('Sorting notifications.') + + sort_integrations(notifications) + + debug('Checking notification ids.') + + notifications, ids = dedupe_integrations(notifications, ids) + + for item in notifications: + for key in NOTIFICATION_RENDER_KEYS: + template = get_jinja_env().get_template(f'{ key }.md') + data = template.render(entry=item) + + if 'variables' in item['meta']: + template = get_jinja_env().from_string(data) + data = template.render(variables=item['meta']['variables']) + + item[key] = data + + item['edit_link'] = make_edit_link(item) + + del item['_src_path'] + del item['_repo'] + del item['_index'] + + return notifications, ids + + +def render_integrations(categories, integrations): + template = get_jinja_env().get_template('integrations.js') + data = template.render( + categories=json.dumps(categories), + integrations=json.dumps(integrations), + ) + OUTPUT_PATH.write_text(data) + + +def main(): + categories = load_categories() + distros = load_yaml(DISTROS_FILE) + collectors = load_collectors() + deploy = load_deploy() + exporters = load_exporters() + notifications = load_notifications() + + collectors, ids = render_collectors(categories, collectors, dict()) + deploy, ids = render_deploy(distros, categories, deploy, ids) + exporters, ids = render_exporters(categories, exporters, ids) + notifications, ids = render_notifications(categories, notifications, ids) + + integrations = collectors + deploy + exporters + notifications + render_integrations(categories, integrations) + + +if __name__ == '__main__': + sys.exit(main()) |