Added support for claiming nodes as part of installation. (#10084)

* Added support for claiming nodes as part of installation.

This adds four new options to the `netdata-installer.sh` script:

* `--claim-token`
* `--claim-rooms`
* `--claim-uri`
* `--claim-proxy`

These directly correspond to the `-token`, `-rooms`, `-uri`, and `-proxy`
options for the `netdata-claim.sh` script. They have the following
associated logic:

* If any are specified and the `--disable-cloud` option is also
  specified, we bail and tell the user to either enable the cloud or
  remove the claiming options.
* If only some but not all of the token, rooms, and uri options  are
  specified, we bail and tell the user that they must pass all three.
* If all three of the token, rooms, and uri are specified, we invoke the
  `netdata-claim.sh` script for the install itself as one of the last
  steps in the installation process, using the values passed to these
  options.

This allows users to directly claim the agent as part of the install,
which is useful for automated installation scenarios.

* Add missing space as suggested by @knatsakis

* Properly handle installs in /.

* Properly handle unprefixed installs.

* Fix another spelling error in an option name.

* Properly fix option naming.

* Move claiming into kickstart script instead of netdata-installer.

This makes us more future-proof.

The required changes also fix some buggy behavior in the option parsing
code in the kickstart scripts.

* Fix checksums.

* Sanely handle the daemon not running during the claiming process.

* Silence incorrect shellcheck warning.

* Simplify condition as suggested by @vkalintiris.

* Clean up old changes that should not be here anymore.

These are leftovers from an earlier revision, they are not actually
needed.

* Add ID generation logic to the claiming script.

This lets it reliably claim nodes which have not yet had the daemon run.

Also fixes a consistency issue in the claiming logic in the Docker
entrypoint.

4 files changed, 194 insertions, 95 deletions

diff --git a/packaging/installer/kickstart-static64.sh b/packaging/installer/kickstart-static64.sh
index 3c45d9efbd..a86a62fcdc 100755
--- a/packaging/installer/kickstart-static64.sh
+++ b/packaging/installer/kickstart-static64.sh
@@ -12,6 +12,10 @@
 #  --local-files              Use a manually provided tarball for the installation
 #  --allow-duplicate-install  do not bail if we detect a duplicate install
 #  --reinstall                if an existing install would be updated, reinstall instead
+#  --claim-token              specify a token to use for claiming the newly installed instance
+#  --claim-url                specify a URL to use for claiming the newly installed isntance
+#  --claim-rooms              specify a list of rooms to claim the newly installed instance to
+#  --claim-proxy              specify a proxy to use while claiming the newly installed instance
 #
 # Environment options:
 #
@@ -224,56 +228,81 @@ NETDATA_INSTALLER_OPTIONS=""
 NETDATA_UPDATES="--auto-update"
 RELEASE_CHANNEL="nightly"
 while [ -n "${1}" ]; do
-  if [ "${1}" = "--dont-wait" ] || [ "${1}" = "--non-interactive" ] || [ "${1}" = "--accept" ]; then
-    opts="${opts} --accept"
-    shift 1
-  elif [ "${1}" = "--dont-start-it" ]; then
-    NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}"
-    shift 1
-  elif [ "${1}" = "--no-updates" ]; then
-    NETDATA_UPDATES=""
-    shift 1
-  elif [ "${1}" = "--auto-update" ]; then
-    true # This is the default behaviour, so ignore it.
-    shift 1
-  elif [ "${1}" = "--stable-channel" ]; then
-    RELEASE_CHANNEL="stable"
-    NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}"
-    shift 1
-  elif [ "${1}" = "--disable-telemetry" ]; then
-    NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}"
-    shift 1
-  elif [ "${1}" = "--local-files" ]; then
-    NETDATA_UPDATES="" # Disable autoupdates if using pre-downloaded files.
-    shift 1
-    if [ -z "${1}" ]; then
-      fatal "Option --local-files requires extra information. The desired tarball full filename is needed"
-    fi
+  case "${1}" in
+    "--dont-wait") opts="${opts} --accept" ;;
+    "--non-interactive") opts="${opts} --accept" ;;
+    "--accept") opts="${opts} --accept" ;;
+    "--dont-start-it")
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}"
+      NETDATA_CLAIM_EXTRA="${NETDATA_CLAIM_EXTRA} -daemon-not-running"
+      ;;
+    "--no-updates") NETDATA_UPDATES="" ;;
+    "--stable-channel")
+      RELEASE_CHANNEL="stable"
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}"
+      ;;
+    "--disable-telemetry") NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }${1}";;
+    "--local-files")
+      NETDATA_UPDATES="" # Disable autoupdates if using pre-downloaded files.
+      if [ -z "${2}" ]; then
+        fatal "Option --local-files requires extra information. The desired tarball full filename is needed"
+      fi
 
-    NETDATA_LOCAL_TARBALL_OVERRIDE="${1}"
-    shift 1
-    if [ -z "${1}" ]; then
-      fatal "Option --local-files requires a pair of the tarball source and the checksum file"
-    fi
+      NETDATA_LOCAL_TARBALL_OVERRIDE="${2}"
 
-    NETDATA_LOCAL_TARBALL_OVERRIDE_CHECKSUM="${1}"
-    shift 1
-  elif [ "${1}" = "--allow-duplicate-install" ]; then
-    NETDATA_ALLOW_DUPLICATE_INSTALL=1
-    shift 1
-  elif [ "${1}" = "--reinstall" ]; then
-    NETDATA_REINSTALL=1
-    shift 1
-  else
-    echo >&2 "Unknown option '${1}' or invalid number of arguments. Please check the README for the available arguments of ${0} and try again"
-    exit 1
-  fi
+      if [ -z "${3}" ]; then
+        fatal "Option --local-files requires a pair of the tarball source and the checksum file"
+      fi
+
+      NETDATA_LOCAL_TARBALL_OVERRIDE_CHECKSUM="${3}"
+      shift 2
+      ;;
+    "--allow-duplicate-install") NETDATA_ALLOW_DUPLICATE_INSTALL=1 ;;
+    "--reinstall") NETDATA_REINSTALL=1 ;;
+    "--claim-token")
+      NETDATA_CLAIM_TOKEN="${2}"
+      shift 1
+      ;;
+    "--claim-rooms")
+      NETDATA_CLAIM_ROOMS="${2}"
+      shift 1
+      ;;
+    "--claim-url")
+      NETDATA_CLAIM_URL="${2}"
+      shift 1
+      ;;
+    "--claim-proxy")
+      NETDATA_CLAIM_EXTRA="${NETDATA_CLAIM_EXTRA} -proxy ${2}"
+      shift 1
+      ;;
+    *)
+      echo >&2 "Unknown option '${1}' or invalid number of arguments. Please check the README for the available arguments of ${0} and try again"
+      exit 1
+  esac
+  shift 1
 done
 
 if [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || [ -n "$DO_NOT_TRACK" ]; then
   NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }--disable-telemtry"
 fi
 
+if [ -n "${NETDATA_DISABLE_CLOUD}" ]; then
+  if [ -n "${NETDATA_CLAIM_TOKEN}" ] || [ -n "${NETDATA_CLAIM_ROOMS}" ] || [ -n "${NETDATA_CLAIM_URL}" ]; then
+    run_failed "Cloud explicitly disabled but automatic claiming requested."
+    run_failed "Either enable Netdata Cloud, or remove the --claim-* options."
+    exit 1
+  fi
+fi
+
+# shellcheck disable=SC2235,SC2030
+if ( [ -z "${NETDATA_CLAIM_TOKEN}" ] && [ -n "${NETDATA_CLAIM_URL}" ] ) || ( [ -n "${NETDATA_CLAIM_TOKEN}" ] && [ -z "${NETDATA_CLAIM_URL}" ] ); then
+  run_failed "Invalid claiming options, both a claiming token and URL must be specified."
+  exit 1
+elif [ -z "${NETDATA_CLAIM_TOKEN}" ] && [ -n "${NETDATA_CLAIM_ROOMS}" ]; then
+  run_failed "Invalid claiming options, claim rooms may only be specified when a token and URL are specified."
+  exit 1
+fi
+
 # Netdata Tarball Base URL (defaults to our Google Storage Bucket)
 [ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
 
@@ -365,4 +394,18 @@ if [ $? -eq 0 ]; then
   fi
 else
   echo >&2 "NOTE: did not remove: ${TMPDIR}/netdata-latest.gz.run"
+  exit 1
+fi
+
+# --------------------------------------------------------------------------------------------------------------------
+
+if [ -n "${NETDATA_CLAIM_TOKEN}" ]; then
+  progress "Attempting to claim agent to ${NETDATA_CLAIM_URL}"
+  NETDATA_CLAIM_PATH=/opt/netdata/bin/netdata-claim.sh
+
+  if "${NETDATA_CLAIM_PATH}" -token=${NETDATA_CLAIM_TOKEN} -rooms=${NETDATA_CLAIM_ROOMS} -url=${NETDATA_CLAIM_URL} ${NETDATA_CLAIM_EXTRA}; then
+    progress "Successfully claimed node"
+  else
+    run_failed "Unable to claim node, you must do so manually."
+  fi
 fi
diff --git a/packaging/installer/kickstart.sh b/packaging/installer/kickstart.sh
index c97587c189..bd18b7db58 100755
--- a/packaging/installer/kickstart.sh
+++ b/packaging/installer/kickstart.sh
@@ -16,6 +16,10 @@
 #  --local-files              set the full path of the desired tarball to run install with
 #  --allow-duplicate-install  do not bail if we detect a duplicate install
 #  --reinstall                if an existing install would be updated, reinstall instead
+#  --claim-token              specify a token to use for claiming the newly installed instance
+#  --claim-url                specify a URL to use for claiming the newly installed isntance
+#  --claim-rooms              specify a list of rooms to claim the newly installed instance to
+#  --claim-proxy              specify a proxy to use while claiming the newly installed instance
 #
 # Environment options:
 #
@@ -317,66 +321,83 @@ NETDATA_INSTALLER_OPTIONS=""
 NETDATA_UPDATES="--auto-update"
 RELEASE_CHANNEL="nightly"
 while [ -n "${1}" ]; do
-  if [ "${1}" = "all" ]; then
-    PACKAGES_INSTALLER_OPTIONS="netdata-all"
-    shift 1
-  elif [ "${1}" = "--dont-wait" ] || [ "${1}" = "--non-interactive" ]; then
-    INTERACTIVE=0
-    shift 1
-  elif [ "${1}" = "--no-updates" ]; then
-    # echo >&2 "netdata will not auto-update"
-    NETDATA_UPDATES=
-    shift 1
-  elif [ "${1}" = "--stable-channel" ]; then
-    RELEASE_CHANNEL="stable"
-    NETDATA_INSTALLER_OPTIONS="$NETDATA_INSTALLER_OPTIONS --stable-channel"
-    shift 1
-  elif [ "${1}" = "--allow-duplicate-install" ]; then
-    NETDATA_ALLOW_DUPLICATE_INSTALL=1
-    shift 1
-  elif [ "${1}" = "--reinstall" ]; then
-    NETDATA_REINSTALL=1
-    shift 1
-  elif [ "${1}" = "--local-files" ]; then
-    shift 1
-    if [ -z "${1}" ]; then
-      fatal "Missing netdata: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
-    fi
+  case "${1}" in
+    "all") PACKAGES_INSTALLER_OPTIONS="netdata-all" ;;
+    "--dont-wait") INTERACTIVE=0 ;;
+    "--non-interactive") INTERACTIVE=0 ;;
+    "--no-updates") NETDATA_UPDATES= ;;
+    "--stable-channel")
+      RELEASE_CHANNEL="stable"
+      NETDATA_INSTALLER_OPTIONS="$NETDATA_INSTALLER_OPTIONS --stable-channel"
+      ;;
+    "--allow-duplicate-install") NETDATA_ALLOW_DUPLICATE_INSTALL=1 ;;
+    "--reinstall") NETDATA_REINSTALL=1 ;;
+    "--claim-token")
+      NETDATA_CLAIM_TOKEN="${2}"
+      shift 1
+      ;;
+    "--claim-rooms")
+      NETDATA_CLAIM_ROOMS="${2}"
+      shift 1
+      ;;
+    "--claim-url")
+      NETDATA_CLAIM_URL="${2}"
+      shift 1
+      ;;
+    "--claim-proxy")
+      NETDATA_CLAIM_EXTRA="${NETDATA_CLAIM_EXTRA} -proxy ${2}"
+      shift 1
+      ;;
+    "--dont-start-it")
+      NETDATA_CLAIM_EXTRA="${NETDATA_CLAIM_EXTRA} -daemon-not-running"
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS} --dont-start-it"
+      ;;
+    "--install")
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS} --install ${2}"
+      NETDATA_PREFIX="${2}"
+      shift 1
+      ;;
+    "--disable-cloud")
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS} --disable-cloud"
+      NETDATA_DISABLE_CLOUD=1
+      ;;
+    "--local-files")
+      if [ -z "${2}" ]; then
+        fatal "Missing netdata: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
+      fi
 
-    export NETDATA_LOCAL_TARBALL_OVERRIDE="${1}"
-    shift 1
+      export NETDATA_LOCAL_TARBALL_OVERRIDE="${2}"
 
-    if [ -z "${1}" ]; then
-      fatal "Missing checksum file: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
-    fi
+      if [ -z "${3}" ]; then
+        fatal "Missing checksum file: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
+      fi
 
-    export NETDATA_LOCAL_TARBALL_OVERRIDE_CHECKSUM="${1}"
-    shift 1
+      export NETDATA_LOCAL_TARBALL_OVERRIDE_CHECKSUM="${3}"
 
-    if [ -z "${1}" ]; then
-      fatal "Missing go.d tarball: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
-    fi
+      if [ -z "${4}" ]; then
+        fatal "Missing go.d tarball: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
+      fi
 
-    export NETDATA_LOCAL_TARBALL_OVERRIDE_GO_PLUGIN="${1}"
-    shift 1
+      export NETDATA_LOCAL_TARBALL_OVERRIDE_GO_PLUGIN="${4}"
 
-    if [ -z "${1}" ]; then
-      fatal "Missing go.d config tarball: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
-    fi
+      if [ -z "${5}" ]; then
+        fatal "Missing go.d config tarball: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
+      fi
 
-    export NETDATA_LOCAL_TARBALL_OVERRIDE_GO_PLUGIN_CONFIG="${1}"
-    shift 1
+      export NETDATA_LOCAL_TARBALL_OVERRIDE_GO_PLUGIN_CONFIG="${5}"
 
-    if [ -z "${1}" ]; then
-      fatal "Missing dependencies management scriptlet: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
-    fi
+      if [ -z "${6}" ]; then
+        fatal "Missing dependencies management scriptlet: Option --local-files requires extra information. The desired tarball for netdata, the checksum, the go.d plugin tarball , the go.d plugin config tarball and the dependency management script, in this particular order"
+      fi
 
-    export NETDATA_LOCAL_TARBALL_OVERRIDE_DEPS_SCRIPT="${1}"
-    shift 1
-  else
-    NETDATA_INSTALLER_OPTIONS="$NETDATA_INSTALLER_OPTIONS ${1}"
-    shift 1
-  fi
+      export NETDATA_LOCAL_TARBALL_OVERRIDE_DEPS_SCRIPT="${6}"
+      shift 5
+      ;;
+    *)
+      NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS} ${1}"
+      ;;
+  esac
+  shift 1
 done
 
 if [ "${INTERACTIVE}" = "0" ]; then
@@ -384,6 +405,23 @@ if [ "${INTERACTIVE}" = "0" ]; then
   NETDATA_INSTALLER_OPTIONS="$NETDATA_INSTALLER_OPTIONS --dont-wait"
 fi
 
+if [ -n "${NETDATA_DISABLE_CLOUD}" ]; then
+  if [ -n "${NETDATA_CLAIM_TOKEN}" ] || [ -n "${NETDATA_CLAIM_ROOMS}" ] || [ -n "${NETDATA_CLAIM_URL}" ]; then
+    run_failed "Cloud explicitly disabled but automatic claiming requested."
+    run_failed "Either enable Netdata Cloud, or remove the --claim-* options."
+    exit 1
+  fi
+fi
+
+# shellcheck disable=SC2235,SC2030
+if ( [ -z "${NETDATA_CLAIM_TOKEN}" ] && [ -n "${NETDATA_CLAIM_URL}" ] ) || ( [ -n "${NETDATA_CLAIM_TOKEN}" ] && [ -z "${NETDATA_CLAIM_URL}" ] ); then
+  run_failed "Invalid claiming options, both a claiming token and URL must be specified."
+  exit 1
+elif [ -z "${NETDATA_CLAIM_TOKEN}" ] && [ -n "${NETDATA_CLAIM_ROOMS}" ]; then
+  run_failed "Invalid claiming options, claim rooms may only be specified when a token and URL are specified."
+  exit 1
+fi
+
 # ---------------------------------------------------------------------------------------------------------------------
 # look for an existing install and try to update that instead if it exists
 
@@ -487,5 +525,23 @@ else
     cd "$(find . -mindepth 1 -maxdepth 1 -type d)" && install "$@"
   else
     fatal "Cannot install netdata from source (the source directory does not include netdata-installer.sh). Leaving all files in ${ndtmpdir}"
+    exit 1
+  fi
+fi
+
+# --------------------------------------------------------------------------------------------------------------------
+
+if [ -n "${NETDATA_CLAIM_TOKEN}" ]; then
+  progress "Attempting to claim agent to ${NETDATA_CLAIM_URL}"
+  if [ -z "${NETDATA_PREFIX}" ] ; then
+    NETDATA_CLAIM_PATH=/usr/sbin/netdata-claim.sh
+  else
+    NETDATA_CLAIM_PATH="${NETDATA_PREFIX}/bin/netdata-claim.sh"
+  fi
+
+  if "${NETDATA_CLAIM_PATH}" -token=${NETDATA_CLAIM_TOKEN} -rooms=${NETDATA_CLAIM_ROOMS} -url=${NETDATA_CLAIM_URL} ${NETDATA_CLAIM_EXTRA}; then
+    progress "Successfully claimed node"
+  else
+    run_failed "Unable to claim node, you must do so manually."
   fi
 fi
diff --git a/packaging/installer/methods/kickstart-64.md b/packaging/installer/methods/kickstart-64.md
index 120cc9e11b..dbb0cd4644 100644
--- a/packaging/installer/methods/kickstart-64.md
+++ b/packaging/installer/methods/kickstart-64.md
@@ -77,7 +77,7 @@ To use `md5sum` to verify the integrity of the `kickstart-static64.sh` script yo
 command above, run the following:
 
 ```bash
-[ "047c86a7c8905955bee39b6980a28e30" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
+[ "33ed36d80c7db0e501b68c4c0c3ceb02" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
 ```
 
 If the script is valid, this command will return `OK, VALID`.
diff --git a/packaging/installer/methods/kickstart.md b/packaging/installer/methods/kickstart.md
index f825f808e7..8ad99dc4a4 100644
--- a/packaging/installer/methods/kickstart.md
+++ b/packaging/installer/methods/kickstart.md
@@ -60,7 +60,7 @@ To use `md5sum` to verify the integrity of the `kickstart.sh` script you will do
 run the following:
 
 ```bash
-[ "8df7a45b2abb336c84507b7c107bcba3" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
+[ "35e92cd3fd8a29621e23962ac5626dfc" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
 ```
 
 If the script is valid, this command will return `OK, VALID`.