Integrate Go plugin with build system. (#17005)

* Integrate Go plugin with build system.

* Fix Debian packaging rules.

* Add FreeBSD support to Go toolchain handling.

* Add Go ldflags handling.

* Fix version detection when GCCGO is used.

* Fix Go ldflags handling.

* Correctly fix Go toolchain version detection.

* Properly mark Go as a required dependency in CMake.

* Disable VCS stamping as it does not work correctly on some platforms.

* Autodetect minimum required Go version from go.mod files.

This allows us to avoid needing to update the CMakeLists.txt file when
the required version changes in a Go component’s go.mod file.

* Prefix GoTools module name with Netdata to ensure we get our local module.

* Update integrations code to use new Go plugin location.

* Remove old go packaging files.

* keep old logic for initial cleanup that is working

* Re-sync Go plugin sources.

* Fix search order for finding Go toolchain.

* update module name

* fix /usr/local/go overwrite condition

---------

Co-authored-by: Fotis Voutsas <fotis@netdata.cloud>
Co-authored-by: ilyam8 <ilya@netdata.cloud>

1 files changed, 17 insertions, 142 deletions

diff --git a/netdata.spec.in b/netdata.spec.in
index ad0a6a36f0..bb9bf279ee 100644
--- a/netdata.spec.in
+++ b/netdata.spec.in
@@ -1,8 +1,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 %global contentdir %{_datadir}/netdata
 %global version @PACKAGE_VERSION@
-%global go_version @GO_PACKAGE_VERSION@
-%global go_name go.d.plugin
 
 # XXX: We are using automatic `Requires:` generation for libraries
 # whenever possible, DO NOT LIST LIBRARY DEPENDENCIES UNLESS THE RESULTANT
@@ -14,6 +12,11 @@ AutoReqProv: yes
 # error.
 %global __os_install_post %{nil}
 
+# This is needed to support proper handling of Go code without requiring
+# external linking or GCCGO, because even recent versions of rpmbuild do
+# not properly support the build IDs generated by the upstream Go toolchain.
+%global _missing_build_ids_terminate_build 0
+
 # Use our custom CMake version from our package builders if we can’t find cmake.
 %if 0%{?centos_ver} == 7
 %global __cmake /cmake/bin/cmake
@@ -39,18 +42,6 @@ AutoReqProv: yes
 %global __cmake_builddir %{__builddir}
 %endif
 
-# Disable go.d.plugin build on outdated golang distros
-%if 0
-%if 0%{?centos_ver:1}
-%if 0%{?centos_ver} >= 10 && 0%{?almalinux_ver:1} && 0%{?rocky_ver:1}
-%global _golang_build 1
-%else
-%global _golang_build 0
-%global _missing_build_ids_terminate_build 0
-%endif
-%endif
-%endif
-
 # Disable eBPF for architectures other than x86
 %ifarch x86_64 i386
 %global _have_ebpf 1
@@ -125,8 +116,6 @@ Release:	1%{?dist}
 License:	GPLv3+
 Group:		Applications/System
 Source0:	https://github.com/%{name}/%{name}/releases/download/%{version}/%{name}-%{version}.tar.gz
-Source1:	https://github.com/%{name}/%{go_name}/releases/download/v%{go_version}/config.tar.gz
-Source2:	https://github.com/%{name}/%{go_name}/archive/refs/tags/v%{go_version}.tar.gz
 URL:		http://my-%{name}.io
 
 # Remove conflicting EPEL packages
@@ -260,11 +249,17 @@ BuildRequires: cups-devel
 # end - cups plugin dependencies
 
 # go.d.plugin dependencies
-%if 0%{?_golang_build}
+#
+# The conditional here is checking for a macro we define in our package
+# builders. If it’s defined, then we’ve injected an upstream copy of
+# the Go toolchain, so we don’t need the package installed (which
+# is needed because Go’s development model is at odds with enterprise
+# distro handling of versioning).
+%if %{?_upstream_go_toolchain:0}%{!?_upstream_go_toolchain:1}
 %if 0%{?suse_version}
-BuildRequires: go
+BuildRequires: go >= 1.21
 %else
-BuildRequires: golang
+BuildRequires: golang >= 1.21
 %endif
 %endif
 # end - go.d.plugin plugin dependencies
@@ -339,15 +334,6 @@ export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging
 %endif
 %endif
 
-# go.d.plugin
-%if 0%{?_golang_build}
-mkdir -p "%{_builddir}/%{go_name}"
-tar -xzf "%{SOURCE1}" -C "%{_builddir}/%{go_name}"
-tar -xzf "%{SOURCE2}" -C "%{_builddir}/%{go_name}"
-cd "%{_builddir}/%{go_name}/%{go_name}-%{go_version}"
-make download
-%endif
-
 %build
 # Conf step
 %cmake -G Ninja \
@@ -408,6 +394,7 @@ make download
 	-DENABLE_PLUGIN_APPS=On \
 	-DENABLE_PLUGIN_CGROUP_NETWORK=On \
 	-DENABLE_PLUGIN_DEBUGFS=On \
+	-DENABLE_PLUGIN_GO=On \
 	-DENABLE_PLUGIN_LOCAL_LISTENERS=On \
 	-DENABLE_PLUGIN_PERF=On \
 	-DENABLE_PLUGIN_SLABINFO=On \
@@ -420,13 +407,6 @@ make download
 # Build step
 %{cmake_build}
 
-# Build go.d.plugin
-%if 0%{?_golang_build}
-cd "%{_builddir}/%{go_name}/%{go_name}-%{go_version}"
-sed -i 's|CGO_ENABLED=0 ||' "hack/go-build.sh"
-TRAVIS_TAG="%{go_version}" %{__make} build
-%endif
-
 %install
 
 # ###########################################################
@@ -471,111 +451,6 @@ install -m 644 -p "%{__cmake_builddir}/system/systemd/%{name}.service.v235" "${R
 install -m 755 -d "${RPM_BUILD_ROOT}%{_presetdir}"
 install -m 644 -p "system/systemd/50-%{name}.preset" "${RPM_BUILD_ROOT}%{_presetdir}/50-%{name}.preset"
 
-# ############################################################
-# Package Go within netdata
-
-# Install builded go.d.plugin
-%if 0%{?_golang_build}
-install -m 0640 "%{_builddir}/%{go_name}/%{go_name}-%{go_version}/bin/godplugin" \
-"${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/%{go_name}"
-cp -r "%{_builddir}/%{go_name}/go.d.conf" "%{_builddir}/%{go_name}/go.d" \
-"${RPM_BUILD_ROOT}%{_libdir}/%{name}/conf.d/"
-%else
-# Install binary go.d.plugin
-safe_sha256sum() {
-	# Within the context of the installer, we only use -c option that is common between the two commands
-	# We will have to reconsider if we start non-common options
-	if command -v sha256sum >/dev/null 2>&1; then
-		sha256sum $@
-	elif command -v shasum >/dev/null 2>&1; then
-		shasum -a 256 $@
-	else
-		fatal "I could not find a suitable checksum binary to use"
-	fi
-}
-
-download_go() {
-	url="${1}"
-	dest="${2}"
-
-	if command -v curl >/dev/null 2>&1; then
-		curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}"
-	elif command -v wget >/dev/null 2>&1; then
-		wget -T 15 -O - "${url}" > "${dest}"
-	else
-		echo >&2
-		echo >&2 "Downloading go.d plugin from '${url}' failed because of missing mandatory packages."
-		echo >&2 "Either add packages or disable it by issuing '--disable-go' in the installer"
-		echo >&2
-		exit 1
-	fi
-}
-
-install_go() {
-	# When updating this value, ensure correct checksums in packaging/go.d.checksums
-	GO_PACKAGE_VERSION="$(cat packaging/go.d.version)"
-	ARCH_MAP=(
-		'i386::386'
-		'i686::386'
-		'x86_64::amd64'
-		'aarch64::arm64'
-		'armv64::arm64'
-		'armv6l::arm'
-		'armv7l::arm'
-		'armv5tel::arm'
-	)
-
-	if [ -z "${NETDATA_DISABLE_GO+x}" ]; then
-		ARCH="%{_arch}"
-		OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-
-		echo >&2 "Install go.d.plugin (ARCH=${ARCH}, OS=${OS})"
-
-		for index in "${ARCH_MAP[@]}" ; do
-			KEY="${index%%::*}"
-			VALUE="${index##*::}"
-			if [ "$KEY" = "$ARCH" ]; then
-				ARCH="${VALUE}"
-				break
-			fi
-		done
-		tmp=$(mktemp -d /tmp/netdata-go-XXXXXX)
-		GO_PACKAGE_BASENAME="go.d.plugin-${GO_PACKAGE_VERSION}.${OS}-${ARCH}.tar.gz"
-		download_go "https://github.com/netdata/go.d.plugin/releases/download/${GO_PACKAGE_VERSION}/${GO_PACKAGE_BASENAME}" "${tmp}/${GO_PACKAGE_BASENAME}"
-		download_go "https://github.com/netdata/go.d.plugin/releases/download/${GO_PACKAGE_VERSION}/config.tar.gz" "${tmp}/config.tar.gz"
-
-		if [ ! -f "${tmp}/${GO_PACKAGE_BASENAME}" ] || [ ! -f "${tmp}/config.tar.gz" ] || [ ! -s "${tmp}/config.tar.gz" ] || [ ! -s "${tmp}/${GO_PACKAGE_BASENAME}" ]; then
-			echo >&2 "Either check the error or consider disabling it by issuing '--disable-go' in the installer"
-			echo >&2
-			return 1
-		fi
-
-		grep "${GO_PACKAGE_BASENAME}\$" "packaging/go.d.checksums" > "${tmp}/sha256sums.txt" 2>/dev/null
-		grep "config.tar.gz" "packaging/go.d.checksums" >> "${tmp}/sha256sums.txt" 2>/dev/null
-
-		# Checksum validation
-		if ! (cd "${tmp}" && safe_sha256sum -c "sha256sums.txt"); then
-
-			echo >&2 "go.d plugin checksum validation failure."
-			echo >&2 "Either check the error or consider disabling it by issuing '--disable-go' in the installer"
-			echo >&2
-
-			echo "go.d.plugin package files checksum validation failed."
-			exit 1
-		fi
-
-		# Install files
-		tar -xf "${tmp}/config.tar.gz" -C "${RPM_BUILD_ROOT}%{_libdir}/%{name}/conf.d/"
-		tar xf "${tmp}/${GO_PACKAGE_BASENAME}"
-		mv "${GO_PACKAGE_BASENAME/\.tar\.gz/}" "go.d.plugin"
-		rm -rf "${tmp}"
-	fi
-	return 0
-}
-install_go
-install -m 0640 -p "%{go_name}" "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/%{go_name}"
-%endif
-
 %if 0%{?_have_ebpf}
 ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-ebpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} \
 ${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d
@@ -729,7 +604,7 @@ rm -rf "${RPM_BUILD_ROOT}"
 %exclude %{_libdir}/%{name}/conf.d/python.d
 
 # Go.d belongs to a different sub-package
-%exclude %{_libexecdir}/%{name}/plugins.d/%{go_name}
+%exclude %{_libexecdir}/%{name}/plugins.d/go.d.plugin
 %exclude %{_libdir}/%{name}/conf.d/go.d.conf
 %exclude %{_libdir}/%{name}/conf.d/go.d
 
@@ -958,7 +833,7 @@ fi
 %defattr(0750,root,netdata,0750)
 # CAP_NET_ADMIN needed for WireGuard collector
 # CAP_NET_RAW needed for ping collector
-%caps(cap_dac_read_search,cap_net_admin,cap_net_raw=eip) %{_libexecdir}/%{name}/plugins.d/%{go_name}
+%caps(cap_dac_read_search,cap_net_admin,cap_net_raw=eip) %{_libexecdir}/%{name}/plugins.d/go.d.plugin
 %defattr(0644,root,netdata,0755)
 %{_libdir}/%{name}/conf.d/go.d.conf
 %{_libdir}/%{name}/conf.d/go.d