Added eBPF collector support to DEB and RPM packages. (#9628)

* Add DEB/RPM package build tests to Travis.

* Add working support for bundling eBPF in binary packages.

* Show stdout and stderr from commands run in LXC.

* Add proper bundling code for libbpf.

* Use AC_CHECK_FILE for libbpf.a external dep.

This way it gets properly logged in both configure output and the
configure log.

10 files changed, 99 insertions, 8 deletions

diff --git a/.travis.yml b/.travis.yml
index 1c35e7eca1..dceed13659 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -165,6 +165,46 @@ jobs:
       script: 'for i in $(seq 0 4); do printf "[XXX: Run #%s]\n" "$i";docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:31" tests/updater_checks.sh && break; done'
       after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 31"
 
+    - name: DEB package test
+      git:
+        depth: false
+      before_install:
+        - sudo apt-get install -y wget lxc python3-lxc python-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl
+      before_script:
+        - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
+      script:
+        - echo "GIT Branch:" && git branch
+        - echo "Last commit:" && git log -1
+        - echo "GIT Describe:" && git describe
+        - echo "packaging/version:" && cat packaging/version
+        - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
+        - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
+        - sudo chown -R root:travis "/var/lib/lxc"
+        - sudo chmod -R 750 "/var/lib/lxc"
+        - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
+      env:
+        - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster"
+        - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
+
+    - name: RPM package test
+      git:
+        depth: false
+      before_install:
+        - sudo apt-get install -y wget lxc lxc-templates python3-lxc python-lxc
+      before_script:
+        - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
+      script:
+        - echo "GIT Branch:" && git branch
+        - echo "Last commit:" && git log -1
+        - echo "GIT Describe:" && git describe
+        - echo "packaging/version:" && cat packaging/version
+        - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
+        - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
+        - sudo chmod -R 755 "/var/lib/lxc"
+        - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
+      env:
+        - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="31" BUILD_STRING="fedora/31"
+        - PACKAGE_TYPE="rpm" REPO_TOOL="dnf"
 
     - stage: Support activities on main branch
       name: Generate changelog for release (only on special and tagged commit msg)
@@ -181,7 +221,6 @@ jobs:
         depth: false
       if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank)
 
-
     # ###### Packaging workflow section ######
     # References:
     # https://us.images.linuxcontainers.org
diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py
index 7999d8be25..61cf358737 100755
--- a/.travis/package_management/common.py
+++ b/.travis/package_management/common.py
@@ -51,7 +51,7 @@ def replace_tag(tag_name, spec, new_tag_content):
 
 def run_command(container, command):
     print("Running command: %s" % command)
-    command_result = container.attach_wait(lxc.attach_run_command, command)
+    command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer)
 
     if command_result != 0:
         raise Exception("Command failed with exit code %d" % command_result)
diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh
index 0c4425fa50..0c00d2fcb2 100644
--- a/.travis/package_management/functions.sh
+++ b/.travis/package_management/functions.sh
@@ -24,8 +24,8 @@ function detect_arch_from_commit {
 			;;
 
 		*)
-			echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. No BUILD_ARCH can be provided"
-			exit 1
+			echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64"
+			export BUILD_ARCH="amd64"
 			;;
 	esac
 
diff --git a/Makefile.am b/Makefile.am
index 8a92f7f41a..e421ab1be0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -72,6 +72,8 @@ dist_noinst_DATA = \
     packaging/mosquitto.version \
     packaging/mosquitto.checksums \
     packaging/bundle-dashboard.sh \
+    packaging/bundle-ebpf.sh \
+    packaging/bundle-libbpf.sh \
     packaging/bundle-mosquitto.sh \
     packaging/check-kernel-config.sh \
     packaging/libbpf.checksums \
diff --git a/configure.ac b/configure.ac
index 6268734922..b3ff4ba785 100644
--- a/configure.ac
+++ b/configure.ac
@@ -960,11 +960,17 @@ AC_CHECK_TYPE(
     [#include <linux/bpf.h>]
 )
 
+AC_CHECK_FILE(
+    externaldeps/libbpf/libbpf.a,
+    [have_libbpf=yes],
+    [have_libbpf=no]
+)
+
 AC_MSG_CHECKING([if ebpf.plugin should be enabled])
 if test "${build_target}" = "linux" -a \
         "${have_libelf}" = "yes" -a \
         "${have_bpf}" = "yes" -a \
-        -f externaldeps/libbpf/libbpf.a; then
+        "${have_libbpf}" = "yes"; then
     OPTIONAL_BPF_CFLAGS="${LIBELF_CFLAGS} -I externaldeps/libbpf/include"
     OPTIONAL_BPF_LIBS="externaldeps/libbpf/libbpf.a ${LIBELF_LIBS}"
     AC_DEFINE([HAVE_LIBBPF], [1], [libbpf usability])
diff --git a/contrib/debian/rules b/contrib/debian/rules
index 97bee36280..b631f36503 100755
--- a/contrib/debian/rules
+++ b/contrib/debian/rules
@@ -36,6 +36,7 @@ override_dh_installinit:
 override_dh_auto_configure:
 	packaging/bundle-mosquitto.sh .
 	packaging/bundle-lws.sh .
+	packaging/bundle-libbpf.sh .
 	autoreconf -ivf
 	dh_auto_configure -- --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=/usr/lib \
 	--libexecdir=/usr/libexec --with-user=netdata --with-math --with-zlib --with-webdir=/var/lib/netdata/www
@@ -76,6 +77,8 @@ override_dh_install:
 		ln -s "/usr/share/netdata/www/$$D" "$(TOP)/var/lib/netdata/www/$$D"; \
 	done
 
+	packaging/bundle-ebpf.sh . ${TOP}/usr/libexec/netdata/plugins.d
+
 	# Install go
 	#
 	debian/install_go.sh $$(cat ${CURDIR}/packaging/go.d.version) $(TOP)/usr/lib/netdata $(TOP)/usr/libexec/netdata
@@ -93,6 +96,9 @@ override_dh_installdocs:
 		--target $(TOP)/usr/share/doc/netdata/ \
 		{} \;
 
+override_dh_shlibdeps:
+	dh_shlibdeps -X libnetdata_ebpf
+
 override_dh_fixperms:
 	dh_fixperms
 
diff --git a/netdata.spec.in b/netdata.spec.in
index 633f76ab29..db1eda44e1 100644
--- a/netdata.spec.in
+++ b/netdata.spec.in
@@ -239,6 +239,7 @@ happened, on your systems and applications.
 %setup -q -n %{name}-%{version}
 export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-mosquitto.sh ${RPM_BUILD_DIR}/%{name}-%{version}
 export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-lws.sh ${RPM_BUILD_DIR}/%{name}-%{version}
+export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-libbpf.sh ${RPM_BUILD_DIR}/%{name}-%{version}
 
 %build
 # Conf step
@@ -412,6 +413,7 @@ install_go
 install -m 0640 -p go.d.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/go.d.plugin"
 
 ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-dashboard.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_datadir}/%{name}/web
+${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-ebpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d
 
 %pre
 
diff --git a/packaging/bundle-dashboard.sh b/packaging/bundle-dashboard.sh
index 8e68ff13fb..9cab084ed2 100755
--- a/packaging/bundle-dashboard.sh
+++ b/packaging/bundle-dashboard.sh
@@ -6,9 +6,9 @@ WEBDIR="${2}"
 DASHBOARD_TARBALL="dashboard.tar.gz"
 DASHBOARD_VERSION="$(cat "${SRCDIR}/packaging/dashboard.version")"
 
-mkdir -p "${SRCDIR}/tmp"
+mkdir -p "${SRCDIR}/tmp/dashboard"
 curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/dashboard/releases/download/${DASHBOARD_VERSION}/${DASHBOARD_TARBALL}" > "${DASHBOARD_TARBALL}" || exit 1
 sha256sum -c "${SRCDIR}/packaging/dashboard.checksums" || exit 1
-tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp" || exit 1
+tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp/dashboard" || exit 1
 # shellcheck disable=SC2046
-cp -a $(find "${SRCDIR}/tmp/build" -mindepth 1 -maxdepth 1) "${WEBDIR}"
+cp -a $(find "${SRCDIR}/tmp/dashboard/build" -mindepth 1 -maxdepth 1) "${WEBDIR}"
diff --git a/packaging/bundle-ebpf.sh b/packaging/bundle-ebpf.sh
new file mode 100755
index 0000000000..ecc365ea44
--- /dev/null
+++ b/packaging/bundle-ebpf.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+SRCDIR="${1}"
+PLUGINDIR="${2}"
+
+EBPF_VERSION="$(cat "${SRCDIR}/packaging/ebpf.version")"
+EBPF_TARBALL="netdata-kernel-collector-glibc-${EBPF_VERSION}.tar.xz"
+
+mkdir -p "${SRCDIR}/tmp/ebpf"
+curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/kernel-collector/releases/download/${EBPF_VERSION}/${EBPF_TARBALL}" > "${EBPF_TARBALL}" || exit 1
+grep "${EBPF_TARBALL}" "${SRCDIR}/packaging/ebpf.checksums" | sha256sum -c - || exit 1
+tar -xaf "${EBPF_TARBALL}" -C "${SRCDIR}/tmp/ebpf" || exit 1
+# shellcheck disable=SC2046
+cp -a $(find "${SRCDIR}/tmp/ebpf" -mindepth 1 -maxdepth 1) "${PLUGINDIR}"
diff --git a/packaging/bundle-libbpf.sh b/packaging/bundle-libbpf.sh
new file mode 100755
index 0000000000..53b8da290d
--- /dev/null
+++ b/packaging/bundle-libbpf.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+LIBBPF_TARBALL="v$(cat "${1}/packaging/libbpf.version").tar.gz"
+LIBBPF_BUILD_PATH="${1}/externaldeps/libbpf/libbpf-$(cat "${1}/packaging/libbpf.version")"
+
+if [ "$(uname -m)" = x86_64 ]; then
+    lib_subdir="lib64"
+else
+    lib_subdir="lib"
+fi
+
+mkdir -p "${1}/externaldeps/libbpf" || exit 1
+curl -sSL --connect-timeout 10 --retry 3 "https://github.com/libbpf/libbpf/archive/${LIBBPF_TARBALL}" > "${LIBBPF_TARBALL}" || exit 1
+sha256sum -c "${1}/packaging/libbpf.checksums" || exit 1
+tar -xzf "${LIBBPF_TARBALL}" -C "${1}/externaldeps/libbpf" || exit 1
+LPWD="${PWD}"
+cd "${LIBBPF_BUILD_PATH}" || exit 1
+patch -p1 < "${1}/libnetdata/ebpf/libbpf.c.diff" || exit 1
+cd "${LPWD}" || exit 1
+make -C "${LIBBPF_BUILD_PATH}/src" BUILD_STATIC_ONLY=1 OBJDIR=build/ DESTDIR=../ install || exit 1
+cp -a "${LIBBPF_BUILD_PATH}/usr/${lib_subdir}/libbpf.a" "${1}/externaldeps/libbpf" || exit 1
+cp -a "${LIBBPF_BUILD_PATH}/usr/include" "${1}/externaldeps/libbpf" || exit 1