From 0542cc3d6bead0535afe5519eea459107c1c0ccd Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Fri, 27 May 2022 07:03:12 -0400 Subject: Cache invariant components in static builds to reduce build times. (#12877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add basic build caching support to static builds. Cache is store din `artifacts/cache/${BUILDARCH}`. Each third-party component utilizes a separate build cache. Invalidation is only done for version changes (more rigorous invalidation is expected to be handled externally). * Integrate static build caching with CI. * Fix fping cache handling. * Test caching in CI. * Properly skip rebuilds on cache hits. * Remove static build container when done with it. * Reuse existing image automatically if it’s for the correct platform. * Test CI build caching. * Fix static build job names. --- packaging/makeself/jobs/20-openssl.install.sh | 29 +++++++-- packaging/makeself/jobs/50-bash-5.1.16.install.sh | 48 ++++++++------- packaging/makeself/jobs/50-curl-7.82.0.install.sh | 71 +++++++++++++---------- packaging/makeself/jobs/50-fping-5.1.install.sh | 37 +++++++----- packaging/makeself/jobs/50-ioping-1.2.install.sh | 15 +++-- 5 files changed, 124 insertions(+), 76 deletions(-) (limited to 'packaging/makeself/jobs') diff --git a/packaging/makeself/jobs/20-openssl.install.sh b/packaging/makeself/jobs/20-openssl.install.sh index ff300447c5..5d9aa068e8 100755 --- a/packaging/makeself/jobs/20-openssl.install.sh +++ b/packaging/makeself/jobs/20-openssl.install.sh @@ -13,15 +13,36 @@ export CFLAGS='-fno-lto -pipe' export LDFLAGS='-static' export PKG_CONFIG="pkg-config --static" -# Might be bind-mounted -if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then +if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then + rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl" +fi + +if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then + rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl" +fi + +cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/openssl" + +if [ -d "${cache}" ]; then + echo "Found cached copy of build directory for openssl, using it." + cp -a "${cache}/openssl" "${NETDATA_MAKESELF_PATH}/tmp/" + CACHE_HIT=1 +else + echo "No cached copy of build directory for openssl found, fetching sources instead." run git clone --branch "${version}" --single-branch --depth 1 git://git.openssl.org/openssl.git "${NETDATA_MAKESELF_PATH}/tmp/openssl" + CACHE_HIT=0 fi + cd "${NETDATA_MAKESELF_PATH}/tmp/openssl" || exit 1 -run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl -run make -j "$(nproc)" +if [ "${CACHE_HIT:-0}" -eq 0 ]; then + run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl + run make -j "$(nproc)" +fi + run make -j "$(nproc)" install_sw +store_cache openssl "${NETDATA_MAKESELF_PATH}/tmp/openssl" + # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::endgroup::" || true diff --git a/packaging/makeself/jobs/50-bash-5.1.16.install.sh b/packaging/makeself/jobs/50-bash-5.1.16.install.sh index aacbe1a4bf..d9f6e5bb0c 100755 --- a/packaging/makeself/jobs/50-bash-5.1.16.install.sh +++ b/packaging/makeself/jobs/50-bash-5.1.16.install.sh @@ -4,37 +4,43 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 +version="5.1.16" + # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building bash" || true -fetch "bash-5.1.16" "http://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz" \ - 5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558 +fetch "bash-${version}" "http://ftp.gnu.org/gnu/bash/bash-${version}.tar.gz" \ + 5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558 bash export CFLAGS="-pipe" export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig" -run ./configure \ - --prefix="${NETDATA_INSTALL_PATH}" \ - --without-bash-malloc \ - --enable-static-link \ - --enable-net-redirections \ - --enable-array-variables \ - --disable-progcomp \ - --disable-profiling \ - --disable-nls \ - --disable-dependency-tracking - -run make clean -run make -j "$(nproc)" - -cat > examples/loadables/Makefile << EOF -all: -clean: -install: -EOF +if [ "${CACHE_HIT:-0}" -eq 0 ]; then + run ./configure \ + --prefix="${NETDATA_INSTALL_PATH}" \ + --without-bash-malloc \ + --enable-static-link \ + --enable-net-redirections \ + --enable-array-variables \ + --disable-progcomp \ + --disable-profiling \ + --disable-nls \ + --disable-dependency-tracking + + run make clean + run make -j "$(nproc)" + + cat > examples/loadables/Makefile <<-EOF + all: + clean: + install: + EOF +fi run make install +store_cache bash "${NETDATA_MAKESELF_PATH}/tmp/bash-${version}" + if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then run strip "${NETDATA_INSTALL_PATH}"/bin/bash fi diff --git a/packaging/makeself/jobs/50-curl-7.82.0.install.sh b/packaging/makeself/jobs/50-curl-7.82.0.install.sh index 2b0201aa48..251d350954 100755 --- a/packaging/makeself/jobs/50-curl-7.82.0.install.sh +++ b/packaging/makeself/jobs/50-curl-7.82.0.install.sh @@ -4,49 +4,56 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 +version="7.82.0" + # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building cURL" || true -fetch "curl-7.82.0" "https://curl.haxx.se/download/curl-7.82.0.tar.gz" \ - 910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce +fetch "curl-${version}" "https://curl.haxx.se/download/curl-${version}.tar.gz" \ + 910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce curl export CFLAGS="-I/openssl-static/include -pipe" export LDFLAGS="-static -L/openssl-static/lib" export PKG_CONFIG="pkg-config --static" export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig" -run autoreconf -fi - -run ./configure \ - --prefix="${NETDATA_INSTALL_PATH}" \ - --enable-optimize \ - --disable-shared \ - --enable-static \ - --enable-http \ - --disable-ldap \ - --disable-ldaps \ - --enable-proxy \ - --disable-dict \ - --disable-telnet \ - --disable-tftp \ - --disable-pop3 \ - --disable-imap \ - --disable-smb \ - --disable-smtp \ - --disable-gopher \ - --enable-ipv6 \ - --enable-cookies \ - --with-ca-fallback \ - --with-openssl \ - --disable-dependency-tracking - -# Curl autoconf does not honour the curl_LDFLAGS environment variable -run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile - -run make clean -run make -j "$(nproc)" +if [ "${CACHE_HIT:-0}" -eq 0 ]; then + run autoreconf -fi + + run ./configure \ + --prefix="${NETDATA_INSTALL_PATH}" \ + --enable-optimize \ + --disable-shared \ + --enable-static \ + --enable-http \ + --disable-ldap \ + --disable-ldaps \ + --enable-proxy \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --enable-ipv6 \ + --enable-cookies \ + --with-ca-fallback \ + --with-openssl \ + --disable-dependency-tracking + + # Curl autoconf does not honour the curl_LDFLAGS environment variable + run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile + + run make clean + run make -j "$(nproc)" +fi + run make install +store_cache curl "${NETDATA_MAKESELF_PATH}/tmp/curl-${version}" + if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then run strip "${NETDATA_INSTALL_PATH}"/bin/curl fi diff --git a/packaging/makeself/jobs/50-fping-5.1.install.sh b/packaging/makeself/jobs/50-fping-5.1.install.sh index 70856e4960..644b5524ad 100755 --- a/packaging/makeself/jobs/50-fping-5.1.install.sh +++ b/packaging/makeself/jobs/50-fping-5.1.install.sh @@ -4,32 +4,39 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 +version="5.1" + # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building fping" || true -fetch "fping-5.1" "https://fping.org/dist/fping-5.1.tar.gz" \ - 1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe +fetch "fping-${version}" "https://fping.org/dist/fping-${version}.tar.gz" \ + 1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe fping export CFLAGS="-static -I/openssl-static/include -pipe" export LDFLAGS="-static -L/openssl-static/lib" export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig" -run ./configure \ - --prefix="${NETDATA_INSTALL_PATH}" \ - --enable-ipv4 \ - --enable-ipv6 \ - --disable-dependency-tracking - -cat > doc/Makefile << EOF -all: -clean: -install: -EOF +if [ "${CACHE_HIT:-0}" -eq 0 ]; then + run ./configure \ + --prefix="${NETDATA_INSTALL_PATH}" \ + --enable-ipv4 \ + --enable-ipv6 \ + --disable-dependency-tracking + + cat > doc/Makefile <<-EOF + all: + clean: + install: + EOF + + run make clean + run make -j "$(nproc)" +fi -run make clean -run make -j "$(nproc)" run make install +store_cache fping "${NETDATA_MAKESELF_PATH}/tmp/fping-${version}" + if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then run strip "${NETDATA_INSTALL_PATH}"/bin/fping fi diff --git a/packaging/makeself/jobs/50-ioping-1.2.install.sh b/packaging/makeself/jobs/50-ioping-1.2.install.sh index 7522ac79c9..160f45b986 100755 --- a/packaging/makeself/jobs/50-ioping-1.2.install.sh +++ b/packaging/makeself/jobs/50-ioping-1.2.install.sh @@ -4,19 +4,26 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 +version='1.2' + # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building ioping" || true -fetch "ioping-1.2" "https://github.com/koct9i/ioping/archive/v1.2.tar.gz" \ - d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75 +fetch "ioping-${version}" "https://github.com/koct9i/ioping/archive/v${version}.tar.gz" \ + d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75 ioping export CFLAGS="-static -pipe" -run make clean -run make -j "$(nproc)" +if [ "${CACHE_HIT:-0}" -eq 0 ]; then + run make clean + run make -j "$(nproc)" +fi + run mkdir -p "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ run install -o root -g root -m 4750 ioping "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ +store_cache ioping "${NETDATA_MAKESELF_PATH}/tmp/ioping-${version}" + if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ioping fi -- cgit v1.2.3