Prefer Protobuf’s own CMake config over CMake's FindProtobuf. (#17128)

* Prefer Protobuf’s own CMake config over CMake's FindProtobuf.

The FindProtobuf CMake module shipped by upstream CMake is broken for
Protobuf version 22.0 and newer because it does not correctly pull in
the new Abseil dependencies. Protobuf itself sometimes ships a CMake
Package Configuration module that _does_ work correctly, so use that in
preference to the Find module shipped with CMake.

Upstream bug reference: https://gitlab.kitware.com/cmake/cmake/-/issues/24321

* Properly handle protoc executable.

* Restructure to explicitly handle fallback case ourselves.

This allows proper handling of compatibility code in a way that actually
works for us without us needing to ship a special module to handle the
compatibility case.

* Switch to bundling protobuf via CMake instead of an external script.

* Fix handling of Protobuf inclusion.

- Add correct include directories for protoc.
- Skip installing protobuf when installing the agent.

* Drop spurious quotation marks.

* Properly fix generator expression for protoc include paths.

* Properly default to bundling protobuf in installer code.

* Disable ASAN in unit tests.

It doesn’t work with the modified protobuf handling and per discussion
with the team is non-critical.

* Change comment based on review.

* Revert "Disable ASAN in unit tests."

This reverts commit 6cb98b1b59c55d639d68694424b98e790ba2f5a7.

* Disable IPMI and NFACCT plugins for unit tests.

We don’t actually have any unit tests for them, and they cause issues
building reliably in the unit testing environment.

* Disable ASAN for Abseil and Protobuf when vendoring them.

* Switch to commit hashes for protobuf/abseil.

* Restructure to better encapsulate protobuf handling as it’s own module.

* Fix up bundled protobuf version handling.

Google has complicated rules for C++ build environment support, so we
really need to be checking compiler versions and not _just_ C++ standard
version.

* Fix warnings about invalid defines.

6 files changed, 253 insertions, 19 deletions

diff --git a/packaging/bundle-protobuf.sh b/packaging/bundle-protobuf.sh
deleted file mode 100755
index d715dfe3db..0000000000
--- a/packaging/bundle-protobuf.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-
-PROTOBUF_TARBALL="protobuf-cpp-$(cat "${1}/packaging/protobuf.version").tar.gz"
-PROTOBUF_BUILD_PATH="${1}/externaldeps/protobuf/protobuf-$(cat "${1}/packaging/protobuf.version")"
-
-mkdir -p "${1}/externaldeps/protobuf" || exit 1
-curl -sSL --connect-timeout 10 --retry 3 "https://github.com/protocolbuffers/protobuf/releases/download/v$(cat "${1}/packaging/protobuf.version")/${PROTOBUF_TARBALL}" > "${PROTOBUF_TARBALL}" || exit 1
-sha256sum -c "${1}/packaging/protobuf.checksums" || exit 1
-tar -xz --no-same-owner -f "${PROTOBUF_TARBALL}" -C "${1}/externaldeps/protobuf" || exit 1
-OLDPWD="${PWD}"
-cd "${PROTOBUF_BUILD_PATH}" || exit 1
-./configure --disable-shared --without-zlib --disable-dependency-tracking --with-pic || exit 1
-make -j "$(nproc)" || exit 1
-cd "${OLDPWD}" || exit 1
-
-cp -a "${PROTOBUF_BUILD_PATH}/src" "${1}/externaldeps/protobuf" || exit 1
diff --git a/packaging/cmake/Modules/NetdataFetchContentExtra.cmake b/packaging/cmake/Modules/NetdataFetchContentExtra.cmake
new file mode 100644
index 0000000000..1de1dcef92
--- /dev/null
+++ b/packaging/cmake/Modules/NetdataFetchContentExtra.cmake
@@ -0,0 +1,27 @@
+# Extra tools for working with FetchContent on older CMake
+#
+# Copyright (c) 2024 Netdata Inc.
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# FetchContent_MakeAvailable_NoInstall
+#
+# Add a sub-project with FetchContent, but with the EXCLUDE_FROM_ALL
+# argument for the add_subdirectory part.
+#
+# CMake 3.28 and newer provide a way to do this with an extra argument
+# on FetchContent_Declare, but older versions need you to implement
+# the logic yourself. Once we no longer support CMake versions older
+# than 3.28, we can get rid of this macro.
+#
+# Unlike FetchContent_MakeAvailble, this only accepts a single project
+# to make available.
+macro(FetchContent_MakeAvailable_NoInstall name)
+    include(FetchContent)
+
+    FetchContent_GetProperties(${name})
+
+    if(NOT ${name}_POPULATED)
+        FetchContent_Populate(${name})
+        add_subdirectory(${${name}_SOURCE_DIR} ${${name}_BINARY_DIR} EXCLUDE_FROM_ALL)
+    endif()
+endmacro()
diff --git a/packaging/cmake/Modules/NetdataProtobuf.cmake b/packaging/cmake/Modules/NetdataProtobuf.cmake
new file mode 100644
index 0000000000..d4ae3aec61
--- /dev/null
+++ b/packaging/cmake/Modules/NetdataProtobuf.cmake
@@ -0,0 +1,225 @@
+# Macros and functions for handling of Protobuf
+#
+# Copyright (c) 2024 Netdata Inc.
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+macro(netdata_protobuf_21_tags)
+        set(PROTOBUF_TAG f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c) # v21.12
+        set(NEED_ABSL False)
+endmacro()
+
+macro(netdata_protobuf_25_tags)
+        set(PROTOBUF_TAG 4a2aef570deb2bfb8927426558701e8bfc26f2a4) # v25.3
+        set(NEED_ABSL True)
+        set(ABSL_TAG 2f9e432cce407ce0ae50676696666f33a77d42ac) # 20240116.1
+endmacro()
+
+# Determine what version of protobuf and abseil to bundle.
+#
+# This is unfortunately very complicated because we support systems
+# older than what Google officially supports for C++.
+macro(netdata_set_bundled_protobuf_tags)
+        netdata_protobuf_21_tags()
+
+        if(NOT USE_CXX_11)
+                if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
+                        if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.3.1)
+                                netdata_protobuf_25_tags()
+                        endif()
+                elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+                        if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0.0)
+                                netdata_protobuf_25_tags()
+                        endif()
+                elseif(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang)
+                        if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12)
+                                netdata_protobuf_25_tags()
+                        endif()
+                endif()
+        endif()
+endmacro()
+
+# Prepare a vendored copy of Protobuf for use with Netdata.
+function(netdata_bundle_protobuf)
+        include(FetchContent)
+        include(NetdataFetchContentExtra)
+
+        netdata_set_bundled_protobuf_tags()
+
+        set(FETCHCONTENT_TRY_FIND_PACKAGE_MODE NEVER)
+
+        string(REPLACE "-fsanitize=address" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+        string(REPLACE "-fsanitize=address" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+
+        # ignore debhelper
+        set(FETCHCONTENT_FULLY_DISCONNECTED Off)
+
+        if(NEED_ABSL)
+                set(ABSL_PROPAGATE_CXX_STD On)
+                set(ABSL_ENABLE_INSTALL Off)
+
+                message(STATUS "Preparing bundled Abseil (required by bundled Protobuf)")
+                FetchContent_Declare(absl
+                        GIT_REPOSITORY https://github.com/abseil/abseil-cpp
+                        GIT_TAG ${ABSL_TAG}
+                )
+                FetchContent_MakeAvailable_NoInstall(absl)
+                message(STATUS "Finished preparing bundled Abseil")
+        endif()
+
+        set(protobuf_INSTALL Off)
+        set(protobuf_BUILD_LIBPROTOC Off)
+        set(protobuf_BUILD_TESTS Off)
+        set(protobuf_BUILD_SHARED_LIBS Off)
+
+        message(STATUS "Preparing bundled Protobuf")
+        FetchContent_Declare(protobuf
+                GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+                GIT_TAG ${PROTOBUF_TAG}
+        )
+        FetchContent_MakeAvailable_NoInstall(protobuf)
+        message(STATUS "Finished preparing bundled Protobuf.")
+
+        set(BUNDLED_PROTOBUF True PARENT_SCOPE)
+endfunction()
+
+# Handle detection of Protobuf
+macro(netdata_detect_protobuf)
+        if(NOT ENABLE_BUNDLED_PROTOBUF)
+                if (NOT BUILD_SHARED_LIBS)
+                        set(Protobuf_USE_STATIC_LIBS On)
+                endif()
+
+                # The FindProtobuf CMake module shipped by upstream CMake is
+                # broken for Protobuf version 22.0 and newer because it does
+                # not correctly pull in the new Abseil dependencies. Protobuf
+                # itself sometimes ships a CMake Package Configuration module
+                # that _does_ work correctly, so use that in preference to the
+                # Find module shipped with CMake.
+                #
+                # The code below works by first attempting to use find_package
+                # in config mode, and then checking for the existence of the
+                # target we actually use that gets defined by the protobuf
+                # CMake Package Configuration Module to determine if that
+                # worked. A bit of extra logic is required in the case of the
+                # config mode working, because some systems ship compatibility
+                # logic for the old FindProtobuf module while others do not.
+                #
+                # Upstream bug reference: https://gitlab.kitware.com/cmake/cmake/-/issues/24321
+                find_package(Protobuf CONFIG)
+
+                if(NOT TARGET protobuf::libprotobuf)
+                        message(STATUS "Could not find Protobuf using Config mode, falling back to Module mode")
+                        find_package(Protobuf REQUIRED)
+                endif()
+        endif()
+
+        if(TARGET protobuf::libprotobuf)
+                if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc)
+                        get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+                                IMPORTED_LOCATION_RELEASE)
+                        if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+                                get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+                                        IMPORTED_LOCATION_RELWITHDEBINFO)
+                        endif()
+                        if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+                                get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+                                        IMPORTED_LOCATION_MINSIZEREL)
+                        endif()
+                        if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+                                get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+                                        IMPORTED_LOCATION_DEBUG)
+                        endif()
+                        if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+                                get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+                                        IMPORTED_LOCATION_NOCONFIG)
+                        endif()
+                        if(NOT Protobuf_PROTOC_EXECUTABLE)
+                                set(Protobuf_PROTOC_EXECUTABLE protobuf::protoc)
+                        endif()
+                endif()
+
+                # It is technically possible that this may still not
+                # be set by this point, so we need to check it and
+                # fail noisily if it isn't because the build won't
+                # work without it.
+                if(NOT Protobuf_PROTOC_EXECUTABLE)
+                        message(FATAL_ERROR "Could not determine the location of the protobuf compiler for the detected version of protobuf.")
+                endif()
+
+                set(NETDATA_PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE})
+                set(NETDATA_PROTOBUF_LIBS protobuf::libprotobuf)
+                get_target_property(NETDATA_PROTOBUF_CFLAGS_OTHER
+                                    protobuf::libprotobuf
+                                    INTERFACE_COMPILE_DEFINITIONS)
+                get_target_property(NETDATA_PROTOBUF_INCLUDE_DIRS
+                                    protobuf::libprotobuf
+                                    INTERFACE_INCLUDE_DIRECTORIES)
+
+                if(NETDATA_PROTOBUF_CFLAGS_OTHER STREQUAL NETDATA_PROTOBUF_CFLAGS_OTHER-NOTFOUND)
+                        set(NETDATA_PROTOBUF_CFLAGS_OTHER "")
+                endif()
+
+                if(NETDATA_PROTOBUF_INCLUDE_DIRS STREQUAL NETDATA_PROTOBUF_INCLUDE_DIRS-NOTFOUND)
+                        set(NETDATA_PROTOBUF_INCLUDE_DIRS "")
+                endif()
+        else()
+                set(NETDATA_PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE})
+                set(NETDATA_PROTOBUF_CFLAGS_OTHER ${PROTOBUF_CFLAGS_OTHER})
+                set(NETDATA_PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS})
+                set(NETDATA_PROTOBUF_LIBS ${PROTOBUF_LIBRARIES})
+        endif()
+
+        set(ENABLE_PROTOBUF True)
+        set(HAVE_PROTOBUF True)
+endmacro()
+
+# Helper function to compile protocol definitions into C++ code.
+function(netdata_protoc_generate_cpp INC_DIR OUT_DIR SRCS HDRS)
+        if(NOT ARGN)
+                message(SEND_ERROR "Error: protoc_generate_cpp() called without any proto files")
+                return()
+        endif()
+
+        set(${INC_DIR})
+        set(${OUT_DIR})
+        set(${SRCS})
+        set(${HDRS})
+
+        foreach(FIL ${ARGN})
+                get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+                get_filename_component(DIR ${ABS_FIL} DIRECTORY)
+                get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+                set(GENERATED_PB_CC "${DIR}/${FIL_WE}.pb.cc")
+                list(APPEND ${SRCS} ${GENERATED_PB_CC})
+
+                set(GENERATED_PB_H "${DIR}/${FIL_WE}.pb.h")
+                list(APPEND ${HDRS} ${GENERATED_PB_H})
+
+                list(APPEND _PROTOC_INCLUDE_DIRS ${INC_DIR})
+
+                if(ENABLE_BUNDLED_PROTOBUF)
+                        list(APPEND _PROTOC_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/_deps/protobuf-src/src/)
+                endif()
+
+                add_custom_command(OUTPUT ${GENERATED_PB_CC} ${GENERATED_PB_H}
+                                   COMMAND ${NETDATA_PROTOBUF_PROTOC_EXECUTABLE}
+                                   ARGS "-I$<JOIN:${_PROTOC_INCLUDE_DIRS},;-I>" --cpp_out=${OUT_DIR} ${ABS_FIL}
+                                   DEPENDS ${ABS_FIL} ${NETDATA_PROTOBUF_PROTOC_EXECUTABLE}
+                                   COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+                                   COMMAND_EXPAND_LISTS)
+        endforeach()
+
+        set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+        set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES COMPILE_OPTIONS -Wno-deprecated-declarations)
+
+        set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+        set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
+
+# Add protobuf to a specified target.
+function(netdata_add_protobuf _target)
+        target_compile_definitions(${_target} PRIVATE ${NETDATA_PROTOBUF_CFLAGS_OTHER})
+        target_include_directories(${_target} PRIVATE ${NETDATA_PROTOBUF_INCLUDE_DIRS})
+        target_link_libraries(${_target} PRIVATE ${NETDATA_PROTOBUF_LIBS})
+endfunction()
diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh
index 63b2f568db..c05382492b 100644
--- a/packaging/installer/functions.sh
+++ b/packaging/installer/functions.sh
@@ -263,7 +263,7 @@ prepare_cmake_options() {
     enable_feature PLUGIN_GO 0
   fi
 
-  if [ "${USE_SYSTEM_PROTOBUF:-1}" -eq 1 ]; then
+  if [ "${USE_SYSTEM_PROTOBUF:-0}" -eq 1 ]; then
     enable_feature BUNDLED_PROTOBUF 0
   else
     enable_feature BUNDLED_PROTOBUF 1
diff --git a/packaging/protobuf.checksums b/packaging/protobuf.checksums
deleted file mode 100644
index 4a025c5fb3..0000000000
--- a/packaging/protobuf.checksums
+++ /dev/null
@@ -1 +0,0 @@
-89ac31a93832e204db6d73b1e80f39f142d5747b290f17340adce5be5b122f94  protobuf-cpp-3.19.4.tar.gz
diff --git a/packaging/protobuf.version b/packaging/protobuf.version
deleted file mode 100644
index de24deecf3..0000000000
--- a/packaging/protobuf.version
+++ /dev/null
@@ -1 +0,0 @@
-3.19.4