Add initial tooling for generating integrations.js file. (#15406)

* Fix link tags in deploy.

* Add initial tooling for generating integrations.js file.

* Skip integrations directory for eslint.

* Add README to explain how to generate integrations.js locally.

* Fix ID/name for top-level categories.

* Deduplicate categories entries.

* Properly render related resources information.

* Warn on and skip bad references for related resources.

* Add CI workflow to rebuild integrations as-needed.

* Add integrations.js to build artifacts.

* Fix actionlint complaints.

* Assorted template fixes.

* Add script to check collector metadata.

* Add default categories for collectors when they have no categories.

* Fix template formatting issues.

* Link related resources properly.

* Skip more sections in rendered output if they are not present in source data.

* Temporarily skip config syntax section.

It needs further work and is not critical at the moment.

* Fix metrics table rendering.

* Hide most overview content if method_description is empty.

* Fix metrics table rendering (again).

* Add detailed description to setup options section.

* Fix detailed description handling for config options.

* Fix config example folding logic.

* Fix multi-instance selection.

* Properly fix multi-instance selection.

* Add titles for labels and metrics charts.

* Include monitored instance name in integration ID.

This is required to disambiguate some ‘virtual’ integrations.

* Indicate if there are no alerts defined for an integration.

* Fix multi-instance in template.

* Improve warning handling in script and fix category handling.

* Hide debug messages by default.

* Fix invalid category name in cgroups plugin.

* Completely fix invalid categories in cgroups plugin.

* Warn about and ignore duplicate integration ids.

* Flag integration type in integrations list.

* Add configuration syntax samples.

* Fix issues in gen_integrations.py

* Validate categories.yaml on load.

* Add support for handling deployment information.

* Fix bugs in gen_integrations.py

* Add code to handle exporters.

* Add link to integrations pointing to their source files.

* Fix table justification.

* Add notification handling to script.

Also tidy up a few other things.

* Fix numerous bugs in gen_integrations.py

* remove trailing space from deploy.yaml command

* make availability one column

* Switch back to multiple columns for availability.

And also switch form +/- to a dot for positive and empty cell for
negative.

* Render setup description.

* Fix platform info rendering in deploy integrations.

* Fix sourcing of cloud-notifications metadata.

* Fix rendering of empty metrics.

* Fix alerts template.

* Add per-instance templating for templated keys.

* Fix go plugin links.

* Fix overview template.

* Fix handling of exporters.

* Fix loading of cloud notification integrations.

* Always show full collector overview.

* Add static troubleshooting content when appropriate.

* Assorted deploy integration updates.

* Add initial copy of integrations.js.

---------

Co-authored-by: Fotis Voutsas <fotis@netdata.cloud>

3 files changed, 92 insertions, 3 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 114eb2c53f..012f75ac57 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -519,6 +519,7 @@ jobs:
           mv ../static-archive/* . || exit 1
           ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
           cp ../packaging/version ./latest-version.txt || exit 1
+          cp ../integrations/integrations.js ./integrations.js || exit 1
           sha256sum -b ./* > sha256sums.txt || exit 1
           cat sha256sums.txt
       - name: Store Artifacts
@@ -753,7 +754,7 @@ jobs:
         with:
           allowUpdates: false
           artifactErrorsFailBuild: true
-          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run'
+          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
           owner: netdata
           repo: netdata-nightlies
           body: Netdata nightly build for ${{ steps.version.outputs.date }}.
@@ -823,7 +824,7 @@ jobs:
         with:
           allowUpdates: false
           artifactErrorsFailBuild: true
-          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run'
+          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
           draft: true
           tag: ${{ needs.normalize-tag.outputs.tag }}
           token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
diff --git a/.github/workflows/generate-integrations.yml b/.github/workflows/generate-integrations.yml
new file mode 100644
index 0000000000..1a18044f20
--- /dev/null
+++ b/.github/workflows/generate-integrations.yml
@@ -0,0 +1,88 @@
+---
+# CI workflow used to regenerate `integrations/integrations.js` when
+# relevant source files are changed.
+name: Generate Integrations
+on:
+  push:
+    branches:
+      - master
+    paths: # If any of these files change, we need to regenerate integrations.js.
+      - 'collectors/**/metadata.yaml'
+      - 'collectors/**/multi_metadata.yaml'
+      - 'integrations/templates/**'
+      - 'integrations/categories.yaml'
+      - 'integrations/gen_integrations.py'
+      - 'packaging/go.d.version'
+  workflow_dispatch: null
+concurrency: # This keeps multiple instances of the job from running concurrently for the same ref.
+  group: integrations-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  generate-integrations:
+    name: Generate Integrations
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Agent
+        id: checkout-agent
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          submodules: recursive
+      - name: Get Go Ref
+        id: get-go-ref
+        run: echo "go_ref=$(cat packaging/go.d.version)" >> "${GITHUB_ENV}"
+      - name: Checkout Go
+        id: checkout-go
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          path: go.d.plugin
+          repository: netdata/go.d.plugin
+          ref: ${{ env.go_ref }}
+      - name: Prepare Dependencies
+        id: prep-deps
+        run: sudo apt-get install python3-jsonschema python3-referencing python3-jinja2 python3-ruamel.yaml
+      - name: Generate Integrations
+        id: generate
+        run: integrations/gen_integrations.py
+      - name: Clean Up Go Repo
+        id: clean-go
+        run: rm -rf go.d.plugin
+      - name: Create PR
+        id: create-pr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
+          commit-message: Regenerate integrations.js
+          branch: integrations-regen
+          title: Regenerate integrations.js
+          body: |
+            Regenerate `integrations/integrations.js` based on the
+            latest code.
+
+            This PR was auto-generated by
+            `.github/workflows/generate-integrations.yml`.
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Integrations regeneration failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: |-
+              ${{ github.repository }}: Failed to create PR rebuilding integrations.js
+              Checkout Agent: ${{ steps.checkout-agent.outcome }}
+              Get Go Ref: ${{ steps.get-go-ref.outcome }}
+              Checkout Go: ${{ steps.checkout-go.outcome }}
+              Prepare Dependencies: ${{ steps.prep-deps.outcome }}
+              Generate Integrations: ${{ steps.generate.outcome }}
+              Clean Up Go Repository: ${{ steps.clean-go.outcome }}
+              Create PR: ${{ steps.create-pr.outcome }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.repository == 'netdata/netdata'
+          }}
diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml
index 6a580afa6c..7e76717edf 100644
--- a/.github/workflows/review.yml
+++ b/.github/workflows/review.yml
@@ -54,7 +54,7 @@ jobs:
         run: |
           if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/eslint') }}" = "true" ]; then
             echo "run=true" >> "${GITHUB_OUTPUT}"
-          elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/v1" | grep -v "web/gui/v2" | grep -Eq '.*\.js|node\.d\.plugin\.in' ; then
+          elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/v1" | grep -v "web/gui/v2" | grep -v "integrations/" | grep -Eq '.*\.js' ; then
             echo "run=true" >> "${GITHUB_OUTPUT}"
             echo 'JS files have changed, need to run ESLint.'
           else