Migrated release build process from Travis CI to GitHub Actions. (#11055)

* Initial version of release build workflow.

* Consolidate static and dist build CI.

* Typo fixes.

* Ensure dependencies are present for dist tarball build.

* Further typo fixes.

* Convert Travis CI to trigger newrelease build workflow.

* Fix actionlint errors.

* Fix distfile name handling.

* Fix handling of dist directory.

6 files changed, 269 insertions, 121 deletions

diff --git a/.github/scripts/run_install_with_dist_file.sh b/.github/scripts/run_install_with_dist_file.sh
index a8a598522e..18cf809274 100755
--- a/.github/scripts/run_install_with_dist_file.sh
+++ b/.github/scripts/run_install_with_dist_file.sh
@@ -18,7 +18,7 @@ shift
 
 printf >&2 "Opening dist archive %s ... " "${distfile}"
 tar -xovf "${distfile}"
-distdir="$(echo "${distfile}" | cut -d. -f1,2,3)"
+distdir="$(echo "${distfile}" | rev | cut -d. -f3- | rev)"
 cp -a packaging/installer/install-required-packages.sh "${distdir}/install-required-packages.sh"
 if [ ! -d "${distdir}" ]; then
   printf >&2 "ERROR: %s is not a directory" "${distdir}"
diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml
index e37893338e..a2607e5bff 100644
--- a/.github/workflows/build-and-install.yml
+++ b/.github/workflows/build-and-install.yml
@@ -11,28 +11,6 @@ concurrency:
   group: builder-${{ github.ref }}
   cancel-in-progress: true
 jobs:
-  static-build:
-    name: Static Build
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch:
-          - x86_64
-          - armv7l
-          - aarch64
-    steps:
-      - name: Git clone repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          submodules: recursive
-      - name: Build
-        run: .github/scripts/build-static.sh ${{ matrix.arch }}
-      - name: Save Packages
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ matrix.arch }}-static-builds
-          path: ${{ github.workspace }}/artifacts/*
   source-build:
     name: Build & Install
     strategy:
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 5397454849..bc746f9ac8 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -53,43 +53,6 @@ jobs:
       - name: Build
         run: |
           docker build -f .github/dockerfiles/Dockerfile.clang .
-  dist-checks:
-    name: Dist
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Prepare environment
-        run: |
-          ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
-          sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \
-                                  libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \
-                                  libnetfilter-acct-dev
-      - name: Configure
-        run: |
-          autoreconf -ivf
-          ./configure \
-            --with-zlib \
-            --with-math \
-            --with-user=netdata \
-            CFLAGS=-O2
-      - name: Make dist
-        run: |
-          make dist
-      - name: Verify & Set distfile
-        run: |
-          ls -lah netdata-*.tar.gz
-          echo "DISTFILE=$(ls netdata-*.tar.gz)" >> $GITHUB_ENV
-      - name: Save Packages
-        uses: actions/upload-artifact@v2
-        with:
-          name: dist-archive
-          path: ${{ github.workspace }}/${{ env.DISTFILE }}
-      - name: Run run_install_with_dist_file.sh
-        run: |
-          ./.github/scripts/run_install_with_dist_file.sh "${DISTFILE}"
   gitignore-check:
     name: .gitignore
     runs-on: ubuntu-latest
diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
new file mode 100644
index 0000000000..559cea01fa
--- /dev/null
+++ b/.github/workflows/release-build.yml
@@ -0,0 +1,240 @@
+---
+# Ci code for building release artifacts.
+name: Release Build
+on:
+  push: # Master branch checks only validate the build and generate artifacts for testing.
+    branches:
+      - master
+  pull_request: null # PR checks only validate the build and generate artifacts for testing.
+  workflow_dispatch:
+    inputs:
+      type:
+        description: Build Type
+        default: nightly
+        required: true
+      version:
+        description: Version Tag
+        default: nightly
+        required: true
+concurrency:
+  group: release-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+jobs:
+  build-dist: # Build the distribution tarball and store it as an artifact.
+    name: Build Distribution Tarball
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Mark Stable
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
+        run: |
+          sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh
+      - name: Build
+        run: |
+          mkdir -p artifacts
+          ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
+          autoreconf -ivf
+          ./configure --prefix=/usr \
+                      --sysconfdir=/etc \
+                      --localstatedir=/var \
+                      --libexecdir=/usr/libexec \
+                      --with-zlib \
+                      --with-math \
+                      --with-user=netdata
+          make dist
+          echo "DISTFILE=$(find . -name 'netdata-*.tar.gz')" >> "$GITHUB_ENV"
+          cp netdata-*.tar.gz artifacts/
+      - name: Test
+        run: |
+          .github/scripts/run_install_with_dist_file.sh "${DISTFILE}"
+      - name: Store
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist-tarball
+          path: artifacts/*.tar.gz
+          retention-days: 30
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Distribution tarball build failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Distribution tarball build failed."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+          }}
+
+  build-static: # Build the static binary archives, and store them as artifacts.
+    name: Build Static
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch:
+          - 'x86_64'
+          - 'armv7l'
+          - 'aarch64'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Mark Stable
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
+        run: |
+          sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh
+      - name: Build
+        run: .github/scripts/build-static.sh ${{ matrix.arch }}
+      - name: Store
+        uses: actions/upload-artifact@v2
+        with:
+          name: static-archive
+          path: artifacts/*.gz.run
+          retention-days: 30
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Static build for ${{ matrix.arch }} failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Static build for ${{ matrix.arch }} failed."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+          }}
+
+  prepare-upload: # Consolidate the artifacts for uploading or releasing.
+    name: Prepare Artifacts
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    needs:
+      - build-dist
+      - build-static
+    steps:
+      - name: Prepare Environment
+        run: mkdir -p artifacts
+      - name: Retrieve Artifacts
+        uses: actions/download-artifact@v2
+      - name: Prepare Artifacts
+        working-directory: ./artifacts/
+        run: |
+          mv ../dist-tarball/* artifacts
+          mv ../static-archive/* artifcts
+          cp ../packaging/version artifacts/latest-version.txt
+          sha256sum -b ./* > "sha256sums.txt"
+      - name: Store Artifacts
+        uses: action/upload-artifact@v2
+        with:
+          name: final-artifacts
+          path: artifacts/*
+          retention-days: 30
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Failed to prepare release artifacts for upload."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+          }}
+
+  upload-nightly: # Upload the nightly build artifacts to GCS.
+    name: Upload Nightly Artifacts
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly'
+    needs:
+      - prepare-upload
+    steps:
+      - name: Retrieve Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: final-artifacts
+      - name: Setup Gcloud
+        uses: google-github-actions/setup-gcloud@v0.2.1
+        with:
+          project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }}
+          service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }}
+          export_default_credentials: true
+      - name: Upload Artifacts
+        uses: google-github-actions/upload-cloud-storage@v0.2.1
+        with:
+          destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }}
+          gzip: false
+          path: ./
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Failed to upload nightly release artifacts:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Failed to upload nightly release artifacts."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+          }}
+
+  upload-release: # Create the draft release and upload the build artifacts.
+    name: Create Release Draft
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
+    needs:
+      - prepare-upload
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Retrieve Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: final-artifacts
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: false
+          artifactErrorsFailBuild: true
+          artifacts: 'sha256sums.txt,netdata-*.tar.gz,netdata-*.gz.run'
+          draft: true
+          tag: ${{ github.event.inputs.version }}
+          token: ${{ secrets.NETDATABOT_TOKEN }}
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Failed to draft release:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Failed to draft release."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name == 'wworkflow_dispatch'
+          }}
diff --git a/.travis.yml b/.travis.yml
index ea297bad10..0900a0e135 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -104,20 +104,11 @@ jobs:
 
     # We only publish if a TAG has been set during packaging
     - stage: Publish for release
-      name: Create release draft
-      git:
-        depth: false
-      env:
-        - RELEASE_CHANNEL=stable
-      before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}"
+      name: Trigger release build and draft release creation
       script:
-        - echo "GIT Branch:" && git branch
-        - echo "Last commit:" && git log -1
-        - echo "GIT Describe:" && git describe
-        - echo "packaging/version:" && cat packaging/version
-        - echo "Generating release artifacts" && .travis/create_artifacts.sh  # Could/should be a common storage to put this and share between jobs
-        - .travis/draft_release.sh
-      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed"
+        - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//')"
+        - .travis/trigger_release_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "release"
+      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger release artifact build during nightly release" "${NOTIF_CHANNEL}"
 
     - name: Trigger Docker image build and publish
       script:
@@ -149,55 +140,11 @@ jobs:
     # This is the nightly execution step
     #
     - stage: Nightly release
-      name: Create nightly release artifacts, publish to GCS
+      name: Trigger nightly artifact build and upload
       script:
-        - echo "GIT Branch:" && git branch
-        - echo "Last commit:" && git log -1
-        - echo "GIT Describe:" && git describe
-        - echo "packaging/version:" && cat packaging/version
-        - .travis/create_artifacts.sh
-      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly artifacts generation failed"
-      git:
-        depth: false
-      before_deploy:
-        echo "Preparing creds under ${TRAVIS_REPO_SLUG}";
-        if [ "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
-           openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d;
-        else
-           echo "Beta deployment stage in progress";
-           openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d;
-        fi;
-      deploy:
-        # Beta storage, used for testing purposes
-        - provider: gcs
-          edge:
-            branch: gcs-ng
-          project_id: netdata-storage
-          credentials: .travis/gcs-credentials.json
-          bucket: "netdata-dev-nightlies"
-          skip_cleanup: true
-          local_dir: "artifacts"
-          on:
-            # Only deploy on netdata/netdata, master branch, when artifacts directory is created
-            repo: ${TRAVIS_REPO_SLUG}
-            branch: master
-            condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} != "netdata/netdata"
-
-        # Production storage
-        - provider: gcs
-          edge:
-            branch: gcs-ng
-          project_id: netdata-storage
-          credentials: .travis/gcs-credentials.json
-          bucket: "netdata-nightlies"
-          skip_cleanup: true
-          local_dir: "artifacts"
-          on:
-            # Only deploy on netdata/netdata, master branch, when artifacts directory is created
-            repo: netdata/netdata
-            branch: master
-            condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata"
-      after_deploy: rm -f .travis/gcs-credentials.json
+        - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//')"
+        - .travis/trigger_release_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "nightly"
+      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger release artifact build during nightly release" "${NOTIF_CHANNEL}"
 
     - name: Trigger Docker image build and publish
       script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "nightly"
diff --git a/.travis/trigger_artifact_build.sh b/.travis/trigger_artifact_build.sh
new file mode 100644
index 0000000000..cbc295340f
--- /dev/null
+++ b/.travis/trigger_artifact_build.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+token="${1}"
+version="${2}"
+type="${3}"
+
+resp="$(curl -X POST \
+             -H 'Accept: application/vnd.github.v3+json' \
+             -H "Authorization: Bearer ${token}" \
+             "https://api.github.com/repos/netdata/netdata/actions/workflows/release-build.yml/dispatches" \
+             -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${type}\"}}")"
+
+if [ -z "${resp}" ]; then
+    echo "Successfully triggered release artifact build."
+    exit 0
+else
+    echo "Failed to trigger release artifact build. Output:"
+    echo "${resp}"
+    exit 1
+fi