From 57c916530c7ce9b7ca7905bfc06d676d34dd2f76 Mon Sep 17 00:00:00 2001 From: James Mills Date: Wed, 3 Jun 2020 00:46:29 +1000 Subject: Fix reliability of kickstart/kickstart-static64 with checksums sometimes failing (#9165) * Add tool to build the dist and static x864_64 artifacts * Add support for overriding the URL for installing/updating from nightlies * Refactor curl/wget usage into less places. Use curl -q and --output and -O options instead of redirecting stdout * Update md5sum(s) of kickstart/kickstart-static64 --- .github/scripts/build-artifacts.sh | 5 +++-- netdata-installer.sh | 22 +-------------------- packaging/installer/functions.sh | 26 +++++++++++++++++++++++-- packaging/installer/kickstart-static64.sh | 30 +++++++++++++++++------------ packaging/installer/kickstart.sh | 18 +++++++++++------ packaging/installer/methods/kickstart-64.md | 2 +- packaging/installer/methods/kickstart.md | 2 +- packaging/installer/netdata-updater.sh | 18 +++++++++++------ 8 files changed, 72 insertions(+), 51 deletions(-) diff --git a/.github/scripts/build-artifacts.sh b/.github/scripts/build-artifacts.sh index e62d9304dc..f816d750ff 100755 --- a/.github/scripts/build-artifacts.sh +++ b/.github/scripts/build-artifacts.sh @@ -16,6 +16,7 @@ prepare_build() { progress "Preparing build" ( test -d artifacts || mkdir -p artifacts + echo "${VERSION}" > packaging/version ) >&2 } @@ -50,8 +51,8 @@ prepare_assets() { cp packaging/version artifacts/latest-version.txt cd artifacts || exit 1 - ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz - ln -s "${BASENAME}.gz.run" netdata-latest.gz.run + ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz + ln -f "${BASENAME}.gz.run" netdata-latest.gz.run sha256sum -b ./* > "sha256sums.txt" ) >&2 } diff --git a/netdata-installer.sh b/netdata-installer.sh index 3eb7b9b47d..71a391db84 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -73,28 +73,8 @@ else source "${NETDATA_SOURCE_DIR}/packaging/installer/functions.sh" || exit 1 fi -download_tarball() { - url="${1}" - dest="${2}" - name="${3}" - opt="${4}" - - if command -v curl > /dev/null 2>&1; then - run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" - elif command -v wget > /dev/null 2>&1; then - run wget -T 15 -O - "${url}" > "${dest}" - else - echo >&2 - echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages." - echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer" - echo >&2 - - run_failed "I need curl or wget to proceed, but neither is available on this system." - fi -} - download_go() { - download_tarball "${1}" "${2}" "go.d plugin" "go" + download_file "${1}" "${2}" "go.d plugin" "go" } # make sure we save all commands we run diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh index af8e08022a..822019c791 100644 --- a/packaging/installer/functions.sh +++ b/packaging/installer/functions.sh @@ -95,7 +95,7 @@ progress() { get() { url="${1}" if command -v curl > /dev/null 2>&1; then - curl -sSL -o - --connect-timeout 10 --retry 3 "${url}" + curl -q -o - -sSL --connect-timeout 10 --retry 3 "${url}" elif command -v wget > /dev/null 2>&1; then wget -T 15 -O - "${url}" else @@ -103,6 +103,28 @@ get() { fi } +download_file() { + url="${1}" + dest="${2}" + name="${3}" + opt="${4}" + + if command -v curl > /dev/null 2>&1; then + run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}" + elif command -v wget > /dev/null 2>&1; then + run wget -T 15 -O "${dest}" "${url}" + else + echo >&2 + echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages." + if [ -n "$opt" ]; then + echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer" + fi + echo >&2 + + run_failed "I need curl or wget to proceed, but neither is available on this system." + fi +} + # ----------------------------------------------------------------------------- # external component handling @@ -114,7 +136,7 @@ fetch_and_verify() { local override=${5} if [ -z "${override}" ]; then - get "${url}" > "${tmp}/${base_name}" + download_file "${url}" "${tmp}/${base_name}" "${component}" else progress "Using provided ${component} archive ${override}" run cp "${override}" "${tmp}/${base_name}" diff --git a/packaging/installer/kickstart-static64.sh b/packaging/installer/kickstart-static64.sh index 673cb860a9..13cba51470 100755 --- a/packaging/installer/kickstart-static64.sh +++ b/packaging/installer/kickstart-static64.sh @@ -11,7 +11,11 @@ # --disable-telemetry Opt-out of anonymous telemetry program (DO_NOT_TRACK=1) # --local-files Use a manually provided tarball for the installation # -# --------------------------------------------------------------------------------------------------------------------- +# Environment options: +# +# NETDATA_TARBALL_BASEURL set the base url for downloading the dist tarball +# +# ---------------------------------------------------------------------------- # library functions copied from packaging/installer/functions.sh setup_terminal() { @@ -43,7 +47,7 @@ setup_terminal() { } setup_terminal || echo > /dev/null -# ----------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- fatal() { printf >&2 "${TPUT_BGRED}${TPUT_WHITE}${TPUT_BOLD} ABORTED ${TPUT_RESET} ${*} \n\n" exit 1 @@ -130,9 +134,9 @@ download() { url="${1}" dest="${2}" if command -v curl > /dev/null 2>&1; then - run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}" + run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}" elif command -v wget > /dev/null 2>&1; then - run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}" + run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}" else fatal "I need curl or wget to proceed, but neither is available on this system." fi @@ -147,13 +151,12 @@ set_tarball_urls() { if [ "$1" = "stable" ]; then local latest # Simple version - # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)" latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)" export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.gz.run" export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt" else - export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.gz.run" - export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt" + export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.gz.run" + export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt" fi } @@ -169,14 +172,14 @@ safe_sha256sum() { fi } -# --------------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- umask 022 sudo="" [ -z "${UID}" ] && UID="$(id -u)" [ "${UID}" -ne "0" ] && sudo="sudo" -# --------------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- if [ "$(uname -m)" != "x86_64" ]; then fatal "Static binary versions of netdata are available only for 64bit Intel/AMD CPUs (x86_64), but yours is: $(uname -m)." fi @@ -185,7 +188,7 @@ if [ "$(uname -s)" != "Linux" ]; then fatal "Static binary versions of netdata are available only for Linux, but this system is $(uname -s)" fi -# --------------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- opts= NETDATA_INSTALLER_OPTIONS="" NETDATA_UPDATES="--auto-update" @@ -235,7 +238,10 @@ if [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || [ -n "$DO_NOT_TRACK" ]; then NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }--disable-telemtry" fi -# --------------------------------------------------------------------------------------------------------------------- +# Netdata Tarball Base URL (defaults to our Google Storage Bucket) +[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies + +# ---------------------------------------------------------------------------- TMPDIR=$(create_tmp_directory) cd "${TMPDIR}" || exit 1 @@ -255,7 +261,7 @@ if ! grep netdata-latest.gz.run "${TMPDIR}/sha256sum.txt" | safe_sha256sum -c - fatal "Static binary checksum validation failed. Stopping netdata installation and leaving binary in ${TMPDIR}" fi -# --------------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- progress "Installing netdata" run ${sudo} sh "${TMPDIR}/netdata-latest.gz.run" ${opts} -- ${NETDATA_UPDATES} ${NETDATA_INSTALLER_OPTIONS} diff --git a/packaging/installer/kickstart.sh b/packaging/installer/kickstart.sh index 9c22083896..a6779174da 100755 --- a/packaging/installer/kickstart.sh +++ b/packaging/installer/kickstart.sh @@ -13,7 +13,11 @@ # --dont-wait do not prompt for user input # --non-interactive do not prompt for user input # --no-updates do not install script for daily updates -# --local-files set the full path of the desired tarball to run install with +# --local-files set the full path of the desired tarball to run install with +# +# Environment options: +# +# NETDATA_TARBALL_BASEURL set the base url for downloading the dist tarball # # This script will: # @@ -29,6 +33,9 @@ # External files PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh" +# Netdata Tarball Base URL (defaults to our Google Storage Bucket) +[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies + # --------------------------------------------------------------------------------------------------------------------- # library functions copied from packaging/installer/functions.sh @@ -158,9 +165,9 @@ download() { url="${1}" dest="${2}" if command -v curl > /dev/null 2>&1; then - run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}" + run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}" elif command -v wget > /dev/null 2>&1; then - run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}" + run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}" else fatal "I need curl or wget to proceed, but neither is available on this system." fi @@ -175,13 +182,12 @@ set_tarball_urls() { if [ "$1" = "stable" ]; then local latest # Simple version - # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)" latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)" export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.tar.gz" export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt" else - export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.tar.gz" - export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt" + export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.tar.gz" + export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt" fi } diff --git a/packaging/installer/methods/kickstart-64.md b/packaging/installer/methods/kickstart-64.md index f76708a535..06b2e16e07 100644 --- a/packaging/installer/methods/kickstart-64.md +++ b/packaging/installer/methods/kickstart-64.md @@ -76,7 +76,7 @@ To use `md5sum` to verify the intregity of the `kickstart-static64.sh` script yo command above, run the following: ```bash -[ "33ecd20452f569c1d9972bcefdf04692" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" +[ "ff717737ccc2212a3363dad7fa8bd20d" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" ``` If the script is valid, this command will return `OK, VALID`. diff --git a/packaging/installer/methods/kickstart.md b/packaging/installer/methods/kickstart.md index 528c7c50b2..658d0167ba 100644 --- a/packaging/installer/methods/kickstart.md +++ b/packaging/installer/methods/kickstart.md @@ -58,7 +58,7 @@ To use `md5sum` to verify the intregity of the `kickstart.sh` script you will do run the following: ```bash -[ "f1e7aa92ca64b797fdc14df59faad205" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" +[ "2057599f8b11ce56f85aa7f26ce7b15b" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" ``` If the script is valid, this command will return `OK, VALID`. diff --git a/packaging/installer/netdata-updater.sh b/packaging/installer/netdata-updater.sh index 781575c711..922d63eaa6 100755 --- a/packaging/installer/netdata-updater.sh +++ b/packaging/installer/netdata-updater.sh @@ -14,6 +14,10 @@ # - NETDATA_TARBALL_CHECKSUM # - NETDATA_PREFIX / NETDATA_LIB_DIR (After 1.16.1 we will only depend on lib dir) # +# Optional environment options: +# +# - NETDATA_TARBALL_BASEURL (set the base url for downloading the dist tarball) +# # Copyright: SPDX-License-Identifier: GPL-3.0-or-later # # Author: Paweł Krupa @@ -108,7 +112,7 @@ get_latest_version() { if [ "${RELEASE_CHANNEL}" == "stable" ]; then latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)" else - latest="$(download "https://storage.googleapis.com/netdata-nightlies/latest-version.txt" /dev/stdout)" + latest="$(download "$NETDATA_TARBALL_BASEURL/latest-version.txt" /dev/stdout)" fi parse_version "$latest" } @@ -123,13 +127,12 @@ set_tarball_urls() { if [ "$1" = "stable" ]; then local latest # Simple version - # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)" latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)" export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.${extension}" export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt" else - export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.${extension}" - export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt" + export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.${extension}" + export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt" fi } @@ -185,8 +188,8 @@ update() { do_not_start="--dont-start-it" fi - if [ -n "${NETDATA_SELECTED_DASHBOARD}" ] ; then - env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}" + if [ -n "${NETDATA_SELECTED_DASHBOARD}" ]; then + env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}" fi info "Re-installing netdata..." @@ -222,6 +225,9 @@ export NETDATA_LIB_DIR="${NETDATA_LIB_DIR:-${NETDATA_PREFIX}/var/lib/netdata}" # Source the tarbal checksum, if not already available from environment (for existing installations with the old logic) [[ -z "${NETDATA_TARBALL_CHECKSUM}" ]] && [[ -f ${NETDATA_LIB_DIR}/netdata.tarball.checksum ]] && NETDATA_TARBALL_CHECKSUM="$(cat "${NETDATA_LIB_DIR}/netdata.tarball.checksum")" +# Netdata Tarball Base URL (defaults to our Google Storage Bucket) +[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies + if [ "${INSTALL_UID}" != "$(id -u)" ]; then fatal "You are running this script as user with uid $(id -u). We recommend to run this script as root (user with uid 0)" fi -- cgit v1.2.3