Cache invariant components in static builds to reduce build times. (#12877)

* Add basic build caching support to static builds.

Cache is store din `artifacts/cache/${BUILDARCH}`. Each third-party
component utilizes a separate build cache. Invalidation is only done for
version changes (more rigorous invalidation is expected to be handled
externally).

* Integrate static build caching with CI.

* Fix fping cache handling.

* Test caching in CI.

* Properly skip rebuilds on cache hits.

* Remove static build container when done with it.

* Reuse existing image automatically if it’s for the correct platform.

* Test CI build caching.

* Fix static build job names.

1 files changed, 45 insertions, 15 deletions

diff --git a/packaging/makeself/functions.sh b/packaging/makeself/functions.sh
index afc8a9ac98..31c28d85c0 100755
--- a/packaging/makeself/functions.sh
+++ b/packaging/makeself/functions.sh
@@ -29,34 +29,64 @@ set -euo pipefail
 # -----------------------------------------------------------------------------
 
 fetch() {
-  local dir="${1}" url="${2}" sha256="${3}"
+  local dir="${1}" url="${2}" sha256="${3}" key="${4}"
   local tar="${dir}.tar.gz"
+  local cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
 
-  if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
-    run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+  if [ -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
+    rm -rf "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
   fi
 
-  # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
-  # two empty spaces between the checksum and the file's path)
-  set +e
-  echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
-  local rc=$?
-  if [ ${rc} -ne 0 ]; then
-      echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
-      echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
-      exit 1
-  fi
-  set -e
+  if [ -d "${cache}/${dir}" ]; then
+    echo "Found cached copy of build directory for ${key}, using it."
+    cp -a "${cache}/${dir}" "${NETDATA_MAKESELF_PATH}/tmp/"
+    CACHE_HIT=1
+  else
+    echo "No cached copy of build directory for ${key} found, fetching sources instead."
+
+    if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
+      run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+    fi
+
+    # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
+    # two empty spaces between the checksum and the file's path)
+    set +e
+    echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
+    local rc=$?
+    if [ ${rc} -ne 0 ]; then
+        echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
+        echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
+        exit 1
+    fi
+    set -e
 
-  if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
     cd "${NETDATA_MAKESELF_PATH}/tmp"
     run tar -zxpf "${tar}"
     cd -
+
+    CACHE_HIT=0
   fi
 
   run cd "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
 }
 
+store_cache() {
+    key="${1}"
+    src="${2}"
+
+    cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
+
+    if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+        if [ -d "${cache}" ]; then
+            rm -rf "${cache}"
+        fi
+
+        mkdir -p "${cache}"
+
+        cp -a "${src}" "${cache}"
+    fi
+}
+
 # -----------------------------------------------------------------------------
 
 # load the functions of the netdata-installer.sh