From 5ce422daf086189fc8a97d77c71d5308e355025a Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Wed, 10 Apr 2024 09:38:44 -0400 Subject: =?UTF-8?q?Skip=20Go=20code=20in=20CI=20if=20it=20hasn=E2=80=99t?= =?UTF-8?q?=20changed.=20(#17077)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Skip building Go components for Docker CI if they have not changed. * Properly handle Go code in general checks PR. * Skip Go code in build checks if it hasn’t changed. * Fix linting issues. * Fix propagation of installer flags. * Fix propagation of environment variables through static build process. * Fix handling of extra install options in static builds. * Skip starting the agent in updater checks. * Fix actionlint warning. --- .github/scripts/build-static.sh | 2 +- .github/scripts/run-updater-check.sh | 2 +- .github/workflows/build.yml | 63 +++++++++++++++------ .github/workflows/checks.yml | 53 ++++++++++++------ .github/workflows/docker.yml | 68 +++++++++++++---------- netdata-installer.sh | 2 +- packaging/makeself/build-static.sh | 11 ++-- packaging/makeself/build.sh | 2 +- packaging/makeself/jobs/70-netdata-git.install.sh | 3 +- packaging/makeself/run-all-jobs.sh | 2 +- 10 files changed, 135 insertions(+), 73 deletions(-) diff --git a/.github/scripts/build-static.sh b/.github/scripts/build-static.sh index e810514388..9b29a3d999 100755 --- a/.github/scripts/build-static.sh +++ b/.github/scripts/build-static.sh @@ -22,7 +22,7 @@ prepare_build() { build_static() { progress "Building static ${BUILDARCH}" ( - USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}" ) >&2 } diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh index 2e70a10aff..456a0e5d07 100755 --- a/.github/scripts/run-updater-check.sh +++ b/.github/scripts/run-updater-check.sh @@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..." /netdata/.github/scripts/ci-support-pkgs.sh mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms. echo ">>> Installing Netdata..." -/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1 +/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1 echo "::group::>>> Pre-Update Environment File Contents" cat /etc/netdata/.environment echo "::endgroup::" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index db7cf94dd7..62e35fd000 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -32,8 +33,8 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files + - name: Check source files + id: check-source-files uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} @@ -44,6 +45,19 @@ jobs: **/*.hh **/*.in **/*.patch + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build files + id: check-build-files + uses: tj-actions/changed-files@v43 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | **/*.cmake CMakeLists.txt netdata-installer.sh @@ -59,29 +73,39 @@ jobs: packaging/*.sh packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md - name: List all changed files in pattern continue-on-error: true env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi build-dist: # Build the distribution tarball and store it as an artifact. name: Build Distribution Tarball @@ -204,7 +228,9 @@ jobs: key: ${{ steps.cache-key.outputs.key }} - name: Build if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs. - run: .github/scripts/build-static.sh ${{ matrix.arch }} + run: | + export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} + .github/scripts/build-static.sh ${{ matrix.arch }} - name: Build if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true' id: build @@ -212,7 +238,9 @@ jobs: with: timeout_minutes: 180 max_attempts: 3 - command: .github/scripts/build-static.sh ${{ matrix.arch }} + command: | + export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} + .github/scripts/build-static.sh ${{ matrix.arch }} - name: Store id: store if: needs.file-check.outputs.run == 'true' @@ -432,19 +460,19 @@ jobs: if: needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build' + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: netdata-installer on ${{ matrix.distro }}, require cloud id: build-cloud if: needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build' + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C id: build-no-jsonc if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build' + /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -545,8 +573,9 @@ jobs: id: updater-check if: needs.file-check.outputs.run == 'true' run: | - docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata test:${{ matrix.artifact_key }} \ - /netdata/.github/scripts/run-updater-check.sh + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \ + -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \ + test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 77ead1942f..bef98a9c34 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,6 +16,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -23,8 +24,8 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files + - name: Check source files + id: check-source-files uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} @@ -35,41 +36,60 @@ jobs: **/*.hh **/*.in **/*.patch + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build files + id: check-build-files + uses: tj-actions/changed-files@v43 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | **/*.cmake CMakeLists.txt .gitignore .github/data/distros.yml .github/workflows/build.yml - .github/scripts/build-static.sh - .github/scripts/get-static-cache-key.sh - .github/scripts/gen-matrix-build.py - .github/scripts/run-updater-check.sh packaging/cmake/ packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md - name: List all changed files in pattern continue-on-error: true env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi libressl-checks: name: LibreSSL @@ -94,7 +114,8 @@ jobs: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata; apk del openssl openssl-dev; apk add libressl libressl-dev protobuf-dev; - ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;' + ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build --disable-go;' + clang-checks: name: Clang needs: @@ -134,7 +155,7 @@ jobs: run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - name: Build netdata if: needs.file-check.outputs.run == 'true' - run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build + run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }} - name: Check that repo is clean if: needs.file-check.outputs.run == 'true' run: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dfe6207c3e..ecbcd02c8c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,6 +31,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -39,8 +40,8 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files + - name: Check source files + id: check-source-files if: github.event_name != 'workflow_dispatch' uses: tj-actions/changed-files@v44 with: @@ -52,7 +53,20 @@ jobs: **/*.hh **/*.in **/*.patch - **/*.cmake + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build system files + id: check-build-files + if: github.event_name != 'workflow_dispatch' + uses: tj-actions/changed-files@v42 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | .dockerignore CMakeLists.txt netdata-installer.sh @@ -66,30 +80,40 @@ jobs: packaging/runtime-check.sh packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md - name: List all changed files in pattern continue-on-error: true if: github.event_name != 'workflow_dispatch' env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi build-images: name: Build Docker Images @@ -143,7 +167,9 @@ jobs: tags: netdata/netdata:test load: true cache-to: type=local,dest=/tmp/build-cache,mode=max - build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + build-args: | + OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }} - name: Test Image id: test if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64' @@ -257,24 +283,8 @@ jobs: with: platforms: ${{ matrix.platform }} cache-from: type=local,src=/tmp/build-cache - build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true - - name: Export Digest - id: export-digest - if: github.repository == 'netdata/netdata' - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - id: upload-digest - if: github.repository == 'netdata/netdata' - uses: actions/upload-artifact@v4 - with: - name: docker-digests-${{ steps.artifact-name.outputs.platform }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: diff --git a/netdata-installer.sh b/netdata-installer.sh index 66696d316d..82acb524de 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -211,7 +211,7 @@ USAGE: ${PROGRAM} [options] --disable-dbengine Explicitly disable DB engine support. --enable-plugin-go Enable the Go plugin. Default: Enabled when possible. --disable-plugin-go Disable the Go plugin. - --disable-go Equivalent to --disable-go-plugin + --disable-go Disable all Go components. --enable-plugin-nfacct Enable nfacct plugin. Default: enable it when libmnl and libnetfilter_acct are available. --disable-plugin-nfacct Explicitly disable the nfacct plugin. --enable-plugin-xenstat Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available. diff --git a/packaging/makeself/build-static.sh b/packaging/makeself/build-static.sh index 260581ed19..7161cfcda1 100755 --- a/packaging/makeself/build-static.sh +++ b/packaging/makeself/build-static.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # SPDX-License-Identifier: GPL-3.0-or-later @@ -54,10 +54,11 @@ fi # Run the build script inside the container if [ -t 1 ]; then run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \ - --platform "${platform}" "${DOCKER_IMAGE_NAME}" \ - /bin/sh /netdata/packaging/makeself/build.sh "${@}" + --platform "${platform}" ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \ + "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}" else run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \ - -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" "${DOCKER_IMAGE_NAME}" \ - /bin/sh /netdata/packaging/makeself/build.sh "${@}" + -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" \ + ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \ + "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}" fi diff --git a/packaging/makeself/build.sh b/packaging/makeself/build.sh index 3ac600ed4a..389e04581b 100755 --- a/packaging/makeself/build.sh +++ b/packaging/makeself/build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash # SPDX-License-Identifier: GPL-3.0-or-later # ----------------------------------------------------------------------------- diff --git a/packaging/makeself/jobs/70-netdata-git.install.sh b/packaging/makeself/jobs/70-netdata-git.install.sh index 0373599a93..13144bfcd1 100755 --- a/packaging/makeself/jobs/70-netdata-git.install.sh +++ b/packaging/makeself/jobs/70-netdata-git.install.sh @@ -37,7 +37,8 @@ run ./netdata-installer.sh \ --dont-scrub-cflags-even-though-it-may-break-things \ --one-time-build \ --disable-logsmanagement \ - --enable-lto + --enable-lto \ + ${EXTRA_INSTALL_FLAGS:+${EXTRA_INSTALL_FLAGS}} \ # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Finishing netdata install" || true diff --git a/packaging/makeself/run-all-jobs.sh b/packaging/makeself/run-all-jobs.sh index dd123c2189..e9b4327bf4 100755 --- a/packaging/makeself/run-all-jobs.sh +++ b/packaging/makeself/run-all-jobs.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # SPDX-License-Identifier: GPL-3.0-or-later set -e -- cgit v1.2.3