From ade4b8798173142fd7cf70be949343217dbe72b0 Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Mon, 16 Aug 2021 13:04:07 -0400 Subject: Added support for bundling protobuf as part of the install. (#11374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for bundling protobu as part of the install. * Fix typo. * Fix `make dist`. * Fix handling of protobuf usage. * Add explicit check for ACLK-NG in builds. * only protos in dist from aclk-schemas Co-authored-by: Timotej Šiškovič --- .github/workflows/build-and-install.yml | 3 + Makefile.am | 19 +++++- aclk/schema-wrappers/schema_wrapper_utils.h | 1 + netdata-installer.sh | 80 +++++++++++++++++++++++- packaging/installer/install-required-packages.sh | 15 +++++ packaging/protobuf.checksums | 1 + packaging/protobuf.version | 1 + 7 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 packaging/protobuf.checksums create mode 100644 packaging/protobuf.version diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml index d1603a888b..87c3588f2e 100644 --- a/.github/workflows/build-and-install.yml +++ b/.github/workflows/build-and-install.yml @@ -124,6 +124,9 @@ jobs: - name: netdata-installer on ${{ matrix.distro }}, require cloud run: | docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud, require ACLK-NG + run: | + docker run -w /netdata -e NETDATA_CONFIGURE_OPTIONS='--with-aclk-ng' test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C if: matrix.rmjsonc != '' run: | diff --git a/Makefile.am b/Makefile.am index aab3101030..8c11b6251a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -57,7 +57,6 @@ dist_noinst_DATA = \ contrib \ docs \ mqtt_websockets \ - aclk/aclk-schemas \ netdata.cppcheck \ netdata.spec \ package.json \ @@ -84,6 +83,8 @@ dist_noinst_DATA = \ packaging/libwebsockets.version \ packaging/mosquitto.checksums \ packaging/mosquitto.version \ + packaging/protobuf.checksums \ + packaging/protobuf.version \ packaging/version \ $(NULL) @@ -599,6 +600,22 @@ ACLK_NG_FILES = \ aclk/schema-wrappers/schema_wrapper_utils.h \ $(NULL) +ACLK_NG_PROTO_DEFINITIONS = \ + aclk/aclk-schemas/proto/aclk/v1/lib.proto \ + aclk/aclk-schemas/proto/agent/v1/connection.proto \ + aclk/aclk-schemas/proto/alarm/v1/config.proto \ + aclk/aclk-schemas/proto/alarm/v1/stream.proto \ + aclk/aclk-schemas/proto/chart/v1/config.proto \ + aclk/aclk-schemas/proto/chart/v1/dimension.proto \ + aclk/aclk-schemas/proto/chart/v1/instance.proto \ + aclk/aclk-schemas/proto/chart/v1/stream.proto \ + aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto \ + aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto \ + aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto \ + $(NULL) + +dist_noinst_DATA += $(ACLK_NG_PROTO_DEFINITIONS) + ACLK_NG_PROTO_BUILT_FILES = aclk/aclk-schemas/proto/agent/v1/connection.pb.cc \ aclk/aclk-schemas/proto/agent/v1/connection.pb.h \ aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.pb.cc \ diff --git a/aclk/schema-wrappers/schema_wrapper_utils.h b/aclk/schema-wrappers/schema_wrapper_utils.h index 61362e3d24..494855f822 100644 --- a/aclk/schema-wrappers/schema_wrapper_utils.h +++ b/aclk/schema-wrappers/schema_wrapper_utils.h @@ -3,6 +3,7 @@ #ifndef SCHEMA_WRAPPER_UTILS_H #define SCHEMA_WRAPPER_UTILS_H +#include #include #if GOOGLE_PROTOBUF_VERSION < 3001000 diff --git a/netdata-installer.sh b/netdata-installer.sh index afc657fa24..900cb15d53 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -236,6 +236,7 @@ USAGE: ${PROGRAM} [options] --disable-lto --disable-x86-sse Disable SSE instructions. By default SSE optimizations are enabled. --use-system-lws Use a system copy of libwebsockets instead of bundling our own (default is to use the bundled copy). + --use-system-protobuf Use a system copy of libprotobuf instead of bundling our own (default is to use the bundled copy). --zlib-is-really-here or --libs-are-really-here If you get errors about missing zlib or libuuid but you know it is available, you might have a broken pkg-config. Use this option to proceed without checking pkg-config. @@ -278,6 +279,7 @@ while [ -n "${1}" ]; do "--zlib-is-really-here") LIBS_ARE_HERE=1 ;; "--libs-are-really-here") LIBS_ARE_HERE=1 ;; "--use-system-lws") USE_SYSTEM_LWS=1 ;; + "--use-system-protobuf") USE_SYSTEM_PROTOBUF=1 ;; "--dont-scrub-cflags-even-though-it-may-break-things") DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS=1 ;; "--dont-start-it") DONOTSTART=1 ;; "--dont-wait") DONOTWAIT=1 ;; @@ -310,7 +312,10 @@ while [ -n "${1}" ]; do "--enable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-kinesis/} --enable-backend-kinesis" ;; "--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-kinesis/} --disable-backend-kinesis" ;; "--enable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-prometheus-remote-write/} --enable-backend-prometheus-remote-write" ;; - "--disable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write" ;; + "--disable-backend-prometheus-remote-write") + NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write" + NETDATA_DISABLE_PROMETHEUS=1 + ;; "--enable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-mongodb/} --enable-backend-mongodb" ;; "--disable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-mongodb/} --disable-backend-mongodb" ;; "--enable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-lto/} --enable-lto" ;; @@ -725,6 +730,79 @@ bundle_libwebsockets # ----------------------------------------------------------------------------- +build_protobuf() { + local env_cmd='' + + if [ -z "${DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS}" ]; then + env_cmd="env CFLAGS=-fPIC CXXFLAGS= LDFLAGS=" + fi + + pushd "${1}" > /dev/null || return 1 + run ${env_cmd} ./configure --disable-shared \ + --without-zlib \ + --disable-dependency-tracking \ + --with-pic || return 1 + run ${env_cmd} ${make} -j$(find_processors) || return 1 + popd > /dev/null || return 1 +} + +copy_protobuf() { + target_dir="${PWD}/externaldeps/protobuf" + + run mkdir -p "${target_dir}" || return 1 + run cp -a "${1}/src" "${target_dir}" || return 1 +} + +bundle_protobuf() { + if [ -n "${NETDATA_DISABLE_CLOUD}" ] && [ -n "${NETDATA_DISABLE_PROMETHEUS}" ]; then + echo "Skipping protobuf" + return 0 + fi + + if [ -n "${USE_SYSTEM_PROTOBUF}" ]; then + echo "Skipping protobuf" + defer_error "You have requested use of a system copy of protobuf. This should work, but it is not recommended as it's very likely to break if you upgrade the currently installed version of protobuf." + return 0 + fi + + PROTOBUF_PACKAGE_VERSION="$(cat packaging/protobuf.version)" + + tmp="$(mktemp -d -t netdata-protobuf-XXXXXX)" + PROTOBUF_PACKAGE_BASENAME="protobuf-cpp-${PROTOBUF_PACKAGE_VERSION}.tar.gz" + + if fetch_and_verify "protobuf" \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_PACKAGE_VERSION}/${PROTOBUF_PACKAGE_BASENAME}" \ + "${PROTOBUF_PACKAGE_BASENAME}" \ + "${tmp}" \ + "${NETDATA_LOCAL_TARBALL_VERRIDE_PROTOBUF}"; then + if run tar -xf "${tmp}/${PROTOBUF_PACKAGE_BASENAME}" -C "${tmp}" && + build_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" && + copy_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" && + rm -rf "${tmp}"; then + run_ok "protobuf built and prepared." + NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS} --with-bundled-protobuf" + else + run_failed "Failed to build protobuf." + if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then + exit 1 + else + defer_error_highlighted "Failed to build protobuf. You may not be able to connect this node to Netdata Cloud." + fi + fi + else + run_failed "Unable to fetch sources for protobuf." + if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then + exit 1 + else + defer_error_highlighted "Unable to fetch sources for protobuf. You may not be able to connect this node to Netdata Cloud." + fi + fi +} + +bundle_protobuf + +# ----------------------------------------------------------------------------- + build_judy() { local env_cmd='' local libtoolize="libtoolize" diff --git a/packaging/installer/install-required-packages.sh b/packaging/installer/install-required-packages.sh index cd1edf08b9..28d11a5b5d 100755 --- a/packaging/installer/install-required-packages.sh +++ b/packaging/installer/install-required-packages.sh @@ -721,6 +721,20 @@ declare -A pkg_gcc=( ['default']="gcc" ) +# g++, required for building protobuf +# All three cases of this not being required are systems that implicitly +# include g++ when installing gcc. +declare -A pkg_gxx=( + ['alpine']="g++" + ['arch']="NOTREQUIRED" + ['clearlinux']="c-basic" + ['debian']="g++" + ['gentoo']="NOTREQUIRED" + ['macos']="NOTREQUIRED" + ['ubuntu']="g++" + ['default']="gcc-c++" +) + declare -A pkg_gdb=( ['gentoo']="sys-devel/gdb" ['macos']="NOTREQUIRED" @@ -1291,6 +1305,7 @@ packages() { require_cmd gcc || require_cmd gcc-multilib || suitable_package gcc + require_cmd g++ || suitable_package gxx require_cmd make || suitable_package make require_cmd autoconf || suitable_package autoconf diff --git a/packaging/protobuf.checksums b/packaging/protobuf.checksums new file mode 100644 index 0000000000..b901a0be4b --- /dev/null +++ b/packaging/protobuf.checksums @@ -0,0 +1 @@ +51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb protobuf-cpp-3.17.3.tar.gz diff --git a/packaging/protobuf.version b/packaging/protobuf.version new file mode 100644 index 0000000000..56cc1b61f8 --- /dev/null +++ b/packaging/protobuf.version @@ -0,0 +1 @@ +3.17.3 -- cgit v1.2.3