Added support for bundling protobuf as part of the install. (#11374)

* Add support for bundling protobu as part of the install.

* Fix typo.

* Fix `make dist`.

* Fix handling of protobuf usage.

* Add explicit check for ACLK-NG in builds.

* only protos in dist from aclk-schemas

Co-authored-by: Timotej Šiškovič <timotej@netdata.cloud>

7 files changed, 118 insertions, 2 deletions

diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml
index d1603a888b..87c3588f2e 100644
--- a/.github/workflows/build-and-install.yml
+++ b/.github/workflows/build-and-install.yml
@@ -124,6 +124,9 @@ jobs:
       - name: netdata-installer on ${{ matrix.distro }}, require cloud
         run: |
           docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
+      - name: netdata-installer on ${{ matrix.distro }}, require cloud, require ACLK-NG
+        run: |
+          docker run -w /netdata -e NETDATA_CONFIGURE_OPTIONS='--with-aclk-ng' test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
       - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
         if: matrix.rmjsonc != ''
         run: |
diff --git a/Makefile.am b/Makefile.am
index aab3101030..8c11b6251a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -57,7 +57,6 @@ dist_noinst_DATA = \
     contrib \
     docs \
     mqtt_websockets \
-    aclk/aclk-schemas \
     netdata.cppcheck \
     netdata.spec \
     package.json \
@@ -84,6 +83,8 @@ dist_noinst_DATA = \
     packaging/libwebsockets.version \
     packaging/mosquitto.checksums \
     packaging/mosquitto.version \
+    packaging/protobuf.checksums \
+    packaging/protobuf.version \
     packaging/version \
     $(NULL)
 
@@ -599,6 +600,22 @@ ACLK_NG_FILES = \
     aclk/schema-wrappers/schema_wrapper_utils.h \
     $(NULL)
 
+ACLK_NG_PROTO_DEFINITIONS = \
+    aclk/aclk-schemas/proto/aclk/v1/lib.proto \
+    aclk/aclk-schemas/proto/agent/v1/connection.proto \
+    aclk/aclk-schemas/proto/alarm/v1/config.proto \
+    aclk/aclk-schemas/proto/alarm/v1/stream.proto \
+    aclk/aclk-schemas/proto/chart/v1/config.proto \
+    aclk/aclk-schemas/proto/chart/v1/dimension.proto \
+    aclk/aclk-schemas/proto/chart/v1/instance.proto \
+    aclk/aclk-schemas/proto/chart/v1/stream.proto \
+    aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto \
+    aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto \
+    aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto \
+    $(NULL)
+
+dist_noinst_DATA += $(ACLK_NG_PROTO_DEFINITIONS)
+
 ACLK_NG_PROTO_BUILT_FILES = aclk/aclk-schemas/proto/agent/v1/connection.pb.cc \
     aclk/aclk-schemas/proto/agent/v1/connection.pb.h \
     aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.pb.cc \
diff --git a/aclk/schema-wrappers/schema_wrapper_utils.h b/aclk/schema-wrappers/schema_wrapper_utils.h
index 61362e3d24..494855f822 100644
--- a/aclk/schema-wrappers/schema_wrapper_utils.h
+++ b/aclk/schema-wrappers/schema_wrapper_utils.h
@@ -3,6 +3,7 @@
 #ifndef SCHEMA_WRAPPER_UTILS_H
 #define SCHEMA_WRAPPER_UTILS_H
 
+#include <sys/time.h>
 #include <google/protobuf/timestamp.pb.h>
 
 #if GOOGLE_PROTOBUF_VERSION < 3001000
diff --git a/netdata-installer.sh b/netdata-installer.sh
index afc657fa24..900cb15d53 100755
--- a/netdata-installer.sh
+++ b/netdata-installer.sh
@@ -236,6 +236,7 @@ USAGE: ${PROGRAM} [options]
   --disable-lto
   --disable-x86-sse          Disable SSE instructions. By default SSE optimizations are enabled.
   --use-system-lws           Use a system copy of libwebsockets instead of bundling our own (default is to use the bundled copy).
+  --use-system-protobuf      Use a system copy of libprotobuf instead of bundling our own (default is to use the bundled copy).
   --zlib-is-really-here or
   --libs-are-really-here     If you get errors about missing zlib or libuuid but you know it is available, you might
                              have a broken pkg-config. Use this option to proceed without checking pkg-config.
@@ -278,6 +279,7 @@ while [ -n "${1}" ]; do
     "--zlib-is-really-here") LIBS_ARE_HERE=1 ;;
     "--libs-are-really-here") LIBS_ARE_HERE=1 ;;
     "--use-system-lws") USE_SYSTEM_LWS=1 ;;
+    "--use-system-protobuf") USE_SYSTEM_PROTOBUF=1 ;;
     "--dont-scrub-cflags-even-though-it-may-break-things") DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS=1 ;;
     "--dont-start-it") DONOTSTART=1 ;;
     "--dont-wait") DONOTWAIT=1 ;;
@@ -310,7 +312,10 @@ while [ -n "${1}" ]; do
     "--enable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-kinesis/} --enable-backend-kinesis" ;;
     "--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-kinesis/} --disable-backend-kinesis" ;;
     "--enable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-prometheus-remote-write/} --enable-backend-prometheus-remote-write" ;;
-    "--disable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write" ;;
+    "--disable-backend-prometheus-remote-write")
+      NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-prometheus-remote-write/} --disable-backend-prometheus-remote-write"
+      NETDATA_DISABLE_PROMETHEUS=1
+      ;;
     "--enable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-backend-mongodb/} --enable-backend-mongodb" ;;
     "--disable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-backend-mongodb/} --disable-backend-mongodb" ;;
     "--enable-lto") NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-lto/} --enable-lto" ;;
@@ -725,6 +730,79 @@ bundle_libwebsockets
 
 # -----------------------------------------------------------------------------
 
+build_protobuf() {
+  local env_cmd=''
+
+  if [ -z "${DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS}" ]; then
+    env_cmd="env CFLAGS=-fPIC CXXFLAGS= LDFLAGS="
+  fi
+
+  pushd "${1}" > /dev/null || return 1
+  run ${env_cmd} ./configure --disable-shared \
+                             --without-zlib \
+                             --disable-dependency-tracking \
+                             --with-pic || return 1
+  run ${env_cmd} ${make} -j$(find_processors) || return 1
+  popd > /dev/null || return 1
+}
+
+copy_protobuf() {
+  target_dir="${PWD}/externaldeps/protobuf"
+
+  run mkdir -p "${target_dir}" || return 1
+  run cp -a "${1}/src" "${target_dir}" || return 1
+}
+
+bundle_protobuf() {
+  if [ -n "${NETDATA_DISABLE_CLOUD}" ] && [ -n "${NETDATA_DISABLE_PROMETHEUS}" ]; then
+    echo "Skipping protobuf"
+    return 0
+  fi
+
+  if [ -n "${USE_SYSTEM_PROTOBUF}" ]; then
+    echo "Skipping protobuf"
+    defer_error "You have requested use of a system copy of protobuf. This should work, but it is not recommended as it's very likely to break if you upgrade the currently installed version of protobuf."
+    return 0
+  fi
+
+  PROTOBUF_PACKAGE_VERSION="$(cat packaging/protobuf.version)"
+
+  tmp="$(mktemp -d -t netdata-protobuf-XXXXXX)"
+  PROTOBUF_PACKAGE_BASENAME="protobuf-cpp-${PROTOBUF_PACKAGE_VERSION}.tar.gz"
+
+  if fetch_and_verify "protobuf" \
+    "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_PACKAGE_VERSION}/${PROTOBUF_PACKAGE_BASENAME}" \
+    "${PROTOBUF_PACKAGE_BASENAME}" \
+    "${tmp}" \
+    "${NETDATA_LOCAL_TARBALL_VERRIDE_PROTOBUF}"; then
+    if run tar -xf "${tmp}/${PROTOBUF_PACKAGE_BASENAME}" -C "${tmp}" &&
+      build_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" &&
+      copy_protobuf "${tmp}/protobuf-${PROTOBUF_PACKAGE_VERSION}" &&
+      rm -rf "${tmp}"; then
+      run_ok "protobuf built and prepared."
+      NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS} --with-bundled-protobuf"
+    else
+      run_failed "Failed to build protobuf."
+      if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then
+        exit 1
+      else
+        defer_error_highlighted "Failed to build protobuf. You may not be able to connect this node to Netdata Cloud."
+      fi
+    fi
+  else
+    run_failed "Unable to fetch sources for protobuf."
+    if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then
+      exit 1
+    else
+      defer_error_highlighted "Unable to fetch sources for protobuf. You may not be able to connect this node to Netdata Cloud."
+    fi
+  fi
+}
+
+bundle_protobuf
+
+# -----------------------------------------------------------------------------
+
 build_judy() {
   local env_cmd=''
   local libtoolize="libtoolize"
diff --git a/packaging/installer/install-required-packages.sh b/packaging/installer/install-required-packages.sh
index cd1edf08b9..28d11a5b5d 100755
--- a/packaging/installer/install-required-packages.sh
+++ b/packaging/installer/install-required-packages.sh
@@ -721,6 +721,20 @@ declare -A pkg_gcc=(
   ['default']="gcc"
 )
 
+# g++, required for building protobuf
+# All three cases of this not being required are systems that implicitly
+# include g++ when installing gcc.
+declare -A pkg_gxx=(
+  ['alpine']="g++"
+  ['arch']="NOTREQUIRED"
+  ['clearlinux']="c-basic"
+  ['debian']="g++"
+  ['gentoo']="NOTREQUIRED"
+  ['macos']="NOTREQUIRED"
+  ['ubuntu']="g++"
+  ['default']="gcc-c++"
+)
+
 declare -A pkg_gdb=(
   ['gentoo']="sys-devel/gdb"
   ['macos']="NOTREQUIRED"
@@ -1291,6 +1305,7 @@ packages() {
 
   require_cmd gcc ||
     require_cmd gcc-multilib || suitable_package gcc
+  require_cmd g++ || suitable_package gxx
 
   require_cmd make || suitable_package make
   require_cmd autoconf || suitable_package autoconf
diff --git a/packaging/protobuf.checksums b/packaging/protobuf.checksums
new file mode 100644
index 0000000000..b901a0be4b
--- /dev/null
+++ b/packaging/protobuf.checksums
@@ -0,0 +1 @@
+51cec99f108b83422b7af1170afd7aeb2dd77d2bcbb7b6bad1f92509e9ccf8cb  protobuf-cpp-3.17.3.tar.gz
diff --git a/packaging/protobuf.version b/packaging/protobuf.version
new file mode 100644
index 0000000000..56cc1b61f8
--- /dev/null
+++ b/packaging/protobuf.version
@@ -0,0 +1 @@
+3.17.3