Fix reliability of kickstart/kickstart-static64 with checksums sometimes failing (#9165)

* Add tool to build the dist and static x864_64 artifacts

* Add support for overriding the URL for installing/updating from nightlies

* Refactor curl/wget usage into less places. Use curl -q and --output and -O options instead of redirecting stdout

* Update md5sum(s) of kickstart/kickstart-static64

8 files changed, 72 insertions, 51 deletions

diff --git a/.github/scripts/build-artifacts.sh b/.github/scripts/build-artifacts.sh
index e62d9304dc..f816d750ff 100755
--- a/.github/scripts/build-artifacts.sh
+++ b/.github/scripts/build-artifacts.sh
@@ -16,6 +16,7 @@ prepare_build() {
   progress "Preparing build"
   (
     test -d artifacts || mkdir -p artifacts
+    echo "${VERSION}" > packaging/version
   ) >&2
 }
 
@@ -50,8 +51,8 @@ prepare_assets() {
     cp packaging/version artifacts/latest-version.txt
 
     cd artifacts || exit 1
-    ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz
-    ln -s "${BASENAME}.gz.run" netdata-latest.gz.run
+    ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz
+    ln -f "${BASENAME}.gz.run" netdata-latest.gz.run
     sha256sum -b ./* > "sha256sums.txt"
   ) >&2
 }
diff --git a/netdata-installer.sh b/netdata-installer.sh
index 3eb7b9b47d..71a391db84 100755
--- a/netdata-installer.sh
+++ b/netdata-installer.sh
@@ -73,28 +73,8 @@ else
   source "${NETDATA_SOURCE_DIR}/packaging/installer/functions.sh" || exit 1
 fi
 
-download_tarball() {
-  url="${1}"
-  dest="${2}"
-  name="${3}"
-  opt="${4}"
-
-  if command -v curl > /dev/null 2>&1; then
-    run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}"
-  elif command -v wget > /dev/null 2>&1; then
-    run wget -T 15 -O - "${url}" > "${dest}"
-  else
-    echo >&2
-    echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages."
-    echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer"
-    echo >&2
-
-    run_failed "I need curl or wget to proceed, but neither is available on this system."
-  fi
-}
-
 download_go() {
-  download_tarball "${1}" "${2}" "go.d plugin" "go"
+  download_file "${1}" "${2}" "go.d plugin" "go"
 }
 
 # make sure we save all commands we run
diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh
index af8e08022a..822019c791 100644
--- a/packaging/installer/functions.sh
+++ b/packaging/installer/functions.sh
@@ -95,7 +95,7 @@ progress() {
 get() {
   url="${1}"
   if command -v curl > /dev/null 2>&1; then
-    curl -sSL -o - --connect-timeout 10 --retry 3 "${url}"
+    curl -q -o - -sSL --connect-timeout 10 --retry 3 "${url}"
   elif command -v wget > /dev/null 2>&1; then
     wget -T 15 -O - "${url}"
   else
@@ -103,6 +103,28 @@ get() {
   fi
 }
 
+download_file() {
+  url="${1}"
+  dest="${2}"
+  name="${3}"
+  opt="${4}"
+
+  if command -v curl > /dev/null 2>&1; then
+    run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
+  elif command -v wget > /dev/null 2>&1; then
+    run wget -T 15 -O "${dest}" "${url}"
+  else
+    echo >&2
+    echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages."
+    if [ -n "$opt" ]; then
+      echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer"
+    fi
+    echo >&2
+
+    run_failed "I need curl or wget to proceed, but neither is available on this system."
+  fi
+}
+
 # -----------------------------------------------------------------------------
 # external component handling
 
@@ -114,7 +136,7 @@ fetch_and_verify() {
   local override=${5}
 
   if [ -z "${override}" ]; then
-    get "${url}" > "${tmp}/${base_name}"
+    download_file "${url}" "${tmp}/${base_name}" "${component}"
   else
     progress "Using provided ${component} archive ${override}"
     run cp "${override}" "${tmp}/${base_name}"
diff --git a/packaging/installer/kickstart-static64.sh b/packaging/installer/kickstart-static64.sh
index 673cb860a9..13cba51470 100755
--- a/packaging/installer/kickstart-static64.sh
+++ b/packaging/installer/kickstart-static64.sh
@@ -11,7 +11,11 @@
 #  --disable-telemetry      Opt-out of anonymous telemetry program (DO_NOT_TRACK=1)
 #  --local-files            Use a manually provided tarball for the installation
 #
-# ---------------------------------------------------------------------------------------------------------------------
+# Environment options:
+#
+#  NETDATA_TARBALL_BASEURL  set the base url for downloading the dist tarball
+#
+# ----------------------------------------------------------------------------
 # library functions copied from packaging/installer/functions.sh
 
 setup_terminal() {
@@ -43,7 +47,7 @@ setup_terminal() {
 }
 setup_terminal || echo > /dev/null
 
-# -----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 fatal() {
   printf >&2 "${TPUT_BGRED}${TPUT_WHITE}${TPUT_BOLD} ABORTED ${TPUT_RESET} ${*} \n\n"
   exit 1
@@ -130,9 +134,9 @@ download() {
   url="${1}"
   dest="${2}"
   if command -v curl > /dev/null 2>&1; then
-    run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}"
+    run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
   elif command -v wget > /dev/null 2>&1; then
-    run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}"
+    run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}"
   else
     fatal "I need curl or wget to proceed, but neither is available on this system."
   fi
@@ -147,13 +151,12 @@ set_tarball_urls() {
   if [ "$1" = "stable" ]; then
     local latest
     # Simple version
-    # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
     latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
     export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.gz.run"
     export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
   else
-    export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.gz.run"
-    export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
+    export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.gz.run"
+    export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
   fi
 }
 
@@ -169,14 +172,14 @@ safe_sha256sum() {
   fi
 }
 
-# ---------------------------------------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 umask 022
 
 sudo=""
 [ -z "${UID}" ] && UID="$(id -u)"
 [ "${UID}" -ne "0" ] && sudo="sudo"
 
-# ---------------------------------------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 if [ "$(uname -m)" != "x86_64" ]; then
   fatal "Static binary versions of netdata are available only for 64bit Intel/AMD CPUs (x86_64), but yours is: $(uname -m)."
 fi
@@ -185,7 +188,7 @@ if [ "$(uname -s)" != "Linux" ]; then
   fatal "Static binary versions of netdata are available only for Linux, but this system is $(uname -s)"
 fi
 
-# ---------------------------------------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 opts=
 NETDATA_INSTALLER_OPTIONS=""
 NETDATA_UPDATES="--auto-update"
@@ -235,7 +238,10 @@ if [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || [ -n "$DO_NOT_TRACK" ]; then
   NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }--disable-telemtry"
 fi
 
-# ---------------------------------------------------------------------------------------------------------------------
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
+
+# ----------------------------------------------------------------------------
 TMPDIR=$(create_tmp_directory)
 cd "${TMPDIR}" || exit 1
 
@@ -255,7 +261,7 @@ if ! grep netdata-latest.gz.run "${TMPDIR}/sha256sum.txt" | safe_sha256sum -c -
   fatal "Static binary checksum validation failed. Stopping netdata installation and leaving binary in ${TMPDIR}"
 fi
 
-# ---------------------------------------------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 progress "Installing netdata"
 run ${sudo} sh "${TMPDIR}/netdata-latest.gz.run" ${opts} -- ${NETDATA_UPDATES} ${NETDATA_INSTALLER_OPTIONS}
 
diff --git a/packaging/installer/kickstart.sh b/packaging/installer/kickstart.sh
index 9c22083896..a6779174da 100755
--- a/packaging/installer/kickstart.sh
+++ b/packaging/installer/kickstart.sh
@@ -13,7 +13,11 @@
 #  --dont-wait                do not prompt for user input
 #  --non-interactive          do not prompt for user input
 #  --no-updates               do not install script for daily updates
-#  --local-files   set the full path of the desired tarball to run install with
+#  --local-files              set the full path of the desired tarball to run install with
+#
+# Environment options:
+#
+#  NETDATA_TARBALL_BASEURL    set the base url for downloading the dist tarball
 #
 # This script will:
 #
@@ -29,6 +33,9 @@
 # External files
 PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"
 
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
+
 # ---------------------------------------------------------------------------------------------------------------------
 # library functions copied from packaging/installer/functions.sh
 
@@ -158,9 +165,9 @@ download() {
   url="${1}"
   dest="${2}"
   if command -v curl > /dev/null 2>&1; then
-    run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}"
+    run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
   elif command -v wget > /dev/null 2>&1; then
-    run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}"
+    run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}"
   else
     fatal "I need curl or wget to proceed, but neither is available on this system."
   fi
@@ -175,13 +182,12 @@ set_tarball_urls() {
   if [ "$1" = "stable" ]; then
     local latest
     # Simple version
-    # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
     latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
     export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.tar.gz"
     export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
   else
-    export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.tar.gz"
-    export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
+    export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.tar.gz"
+    export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
   fi
 }
 
diff --git a/packaging/installer/methods/kickstart-64.md b/packaging/installer/methods/kickstart-64.md
index f76708a535..06b2e16e07 100644
--- a/packaging/installer/methods/kickstart-64.md
+++ b/packaging/installer/methods/kickstart-64.md
@@ -76,7 +76,7 @@ To use `md5sum` to verify the intregity of the `kickstart-static64.sh` script yo
 command above, run the following:
 
 ```bash
-[ "33ecd20452f569c1d9972bcefdf04692" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
+[ "ff717737ccc2212a3363dad7fa8bd20d" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
 ```
 
 If the script is valid, this command will return `OK, VALID`.
diff --git a/packaging/installer/methods/kickstart.md b/packaging/installer/methods/kickstart.md
index 528c7c50b2..658d0167ba 100644
--- a/packaging/installer/methods/kickstart.md
+++ b/packaging/installer/methods/kickstart.md
@@ -58,7 +58,7 @@ To use `md5sum` to verify the intregity of the `kickstart.sh` script you will do
 run the following:
 
 ```bash
-[ "f1e7aa92ca64b797fdc14df59faad205" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
+[ "2057599f8b11ce56f85aa7f26ce7b15b" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
 ```
 
 If the script is valid, this command will return `OK, VALID`.
diff --git a/packaging/installer/netdata-updater.sh b/packaging/installer/netdata-updater.sh
index 781575c711..922d63eaa6 100755
--- a/packaging/installer/netdata-updater.sh
+++ b/packaging/installer/netdata-updater.sh
@@ -14,6 +14,10 @@
 #  - NETDATA_TARBALL_CHECKSUM
 #  - NETDATA_PREFIX / NETDATA_LIB_DIR (After 1.16.1 we will only depend on lib dir)
 #
+# Optional environment options:
+#
+#  - NETDATA_TARBALL_BASEURL (set the base url for downloading the dist tarball)
+#
 # Copyright: SPDX-License-Identifier: GPL-3.0-or-later
 #
 # Author: Paweł Krupa <paulfantom@gmail.com>
@@ -108,7 +112,7 @@ get_latest_version() {
   if [ "${RELEASE_CHANNEL}" == "stable" ]; then
     latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
   else
-    latest="$(download "https://storage.googleapis.com/netdata-nightlies/latest-version.txt" /dev/stdout)"
+    latest="$(download "$NETDATA_TARBALL_BASEURL/latest-version.txt" /dev/stdout)"
   fi
   parse_version "$latest"
 }
@@ -123,13 +127,12 @@ set_tarball_urls() {
   if [ "$1" = "stable" ]; then
     local latest
     # Simple version
-    # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
     latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
     export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.${extension}"
     export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
   else
-    export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.${extension}"
-    export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
+    export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.${extension}"
+    export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
   fi
 }
 
@@ -185,8 +188,8 @@ update() {
       do_not_start="--dont-start-it"
     fi
 
-    if [ -n "${NETDATA_SELECTED_DASHBOARD}" ] ; then
-        env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
+    if [ -n "${NETDATA_SELECTED_DASHBOARD}" ]; then
+      env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
     fi
 
     info "Re-installing netdata..."
@@ -222,6 +225,9 @@ export NETDATA_LIB_DIR="${NETDATA_LIB_DIR:-${NETDATA_PREFIX}/var/lib/netdata}"
 # Source the tarbal checksum, if not already available from environment (for existing installations with the old logic)
 [[ -z "${NETDATA_TARBALL_CHECKSUM}" ]] && [[ -f ${NETDATA_LIB_DIR}/netdata.tarball.checksum ]] && NETDATA_TARBALL_CHECKSUM="$(cat "${NETDATA_LIB_DIR}/netdata.tarball.checksum")"
 
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
+
 if [ "${INSTALL_UID}" != "$(id -u)" ]; then
   fatal "You are running this script as user with uid $(id -u). We recommend to run this script as root (user with uid 0)"
 fi