From 8a105bad8188b7581c92811c069676d3110f130b Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Thu, 30 Jul 2020 07:09:59 -0400 Subject: Added eBPF collector support to DEB and RPM packages. (#9628) * Add DEB/RPM package build tests to Travis. * Add working support for bundling eBPF in binary packages. * Show stdout and stderr from commands run in LXC. * Add proper bundling code for libbpf. * Use AC_CHECK_FILE for libbpf.a external dep. This way it gets properly logged in both configure output and the configure log. --- .travis.yml | 41 ++++++++++++++++++++++++++++++++- .travis/package_management/common.py | 2 +- .travis/package_management/functions.sh | 4 ++-- Makefile.am | 2 ++ configure.ac | 8 ++++++- contrib/debian/rules | 6 +++++ netdata.spec.in | 2 ++ packaging/bundle-dashboard.sh | 6 ++--- packaging/bundle-ebpf.sh | 14 +++++++++++ packaging/bundle-libbpf.sh | 22 ++++++++++++++++++ 10 files changed, 99 insertions(+), 8 deletions(-) create mode 100755 packaging/bundle-ebpf.sh create mode 100755 packaging/bundle-libbpf.sh diff --git a/.travis.yml b/.travis.yml index 1c35e7eca1..dceed13659 100644 --- a/.travis.yml +++ b/.travis.yml @@ -165,6 +165,46 @@ jobs: script: 'for i in $(seq 0 4); do printf "[XXX: Run #%s]\n" "$i";docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:31" tests/updater_checks.sh && break; done' after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 31" + - name: DEB package test + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc python3-lxc python-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chown -R root:travis "/var/lib/lxc" + - sudo chmod -R 750 "/var/lib/lxc" + - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + + - name: RPM package test + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc lxc-templates python3-lxc python-lxc + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="31" BUILD_STRING="fedora/31" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - stage: Support activities on main branch name: Generate changelog for release (only on special and tagged commit msg) @@ -181,7 +221,6 @@ jobs: depth: false if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank) - # ###### Packaging workflow section ###### # References: # https://us.images.linuxcontainers.org diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py index 7999d8be25..61cf358737 100755 --- a/.travis/package_management/common.py +++ b/.travis/package_management/common.py @@ -51,7 +51,7 @@ def replace_tag(tag_name, spec, new_tag_content): def run_command(container, command): print("Running command: %s" % command) - command_result = container.attach_wait(lxc.attach_run_command, command) + command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer) if command_result != 0: raise Exception("Command failed with exit code %d" % command_result) diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh index 0c4425fa50..0c00d2fcb2 100644 --- a/.travis/package_management/functions.sh +++ b/.travis/package_management/functions.sh @@ -24,8 +24,8 @@ function detect_arch_from_commit { ;; *) - echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. No BUILD_ARCH can be provided" - exit 1 + echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64" + export BUILD_ARCH="amd64" ;; esac diff --git a/Makefile.am b/Makefile.am index 8a92f7f41a..e421ab1be0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -72,6 +72,8 @@ dist_noinst_DATA = \ packaging/mosquitto.version \ packaging/mosquitto.checksums \ packaging/bundle-dashboard.sh \ + packaging/bundle-ebpf.sh \ + packaging/bundle-libbpf.sh \ packaging/bundle-mosquitto.sh \ packaging/check-kernel-config.sh \ packaging/libbpf.checksums \ diff --git a/configure.ac b/configure.ac index 6268734922..b3ff4ba785 100644 --- a/configure.ac +++ b/configure.ac @@ -960,11 +960,17 @@ AC_CHECK_TYPE( [#include ] ) +AC_CHECK_FILE( + externaldeps/libbpf/libbpf.a, + [have_libbpf=yes], + [have_libbpf=no] +) + AC_MSG_CHECKING([if ebpf.plugin should be enabled]) if test "${build_target}" = "linux" -a \ "${have_libelf}" = "yes" -a \ "${have_bpf}" = "yes" -a \ - -f externaldeps/libbpf/libbpf.a; then + "${have_libbpf}" = "yes"; then OPTIONAL_BPF_CFLAGS="${LIBELF_CFLAGS} -I externaldeps/libbpf/include" OPTIONAL_BPF_LIBS="externaldeps/libbpf/libbpf.a ${LIBELF_LIBS}" AC_DEFINE([HAVE_LIBBPF], [1], [libbpf usability]) diff --git a/contrib/debian/rules b/contrib/debian/rules index 97bee36280..b631f36503 100755 --- a/contrib/debian/rules +++ b/contrib/debian/rules @@ -36,6 +36,7 @@ override_dh_installinit: override_dh_auto_configure: packaging/bundle-mosquitto.sh . packaging/bundle-lws.sh . + packaging/bundle-libbpf.sh . autoreconf -ivf dh_auto_configure -- --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=/usr/lib \ --libexecdir=/usr/libexec --with-user=netdata --with-math --with-zlib --with-webdir=/var/lib/netdata/www @@ -76,6 +77,8 @@ override_dh_install: ln -s "/usr/share/netdata/www/$$D" "$(TOP)/var/lib/netdata/www/$$D"; \ done + packaging/bundle-ebpf.sh . ${TOP}/usr/libexec/netdata/plugins.d + # Install go # debian/install_go.sh $$(cat ${CURDIR}/packaging/go.d.version) $(TOP)/usr/lib/netdata $(TOP)/usr/libexec/netdata @@ -93,6 +96,9 @@ override_dh_installdocs: --target $(TOP)/usr/share/doc/netdata/ \ {} \; +override_dh_shlibdeps: + dh_shlibdeps -X libnetdata_ebpf + override_dh_fixperms: dh_fixperms diff --git a/netdata.spec.in b/netdata.spec.in index 633f76ab29..db1eda44e1 100644 --- a/netdata.spec.in +++ b/netdata.spec.in @@ -239,6 +239,7 @@ happened, on your systems and applications. %setup -q -n %{name}-%{version} export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-mosquitto.sh ${RPM_BUILD_DIR}/%{name}-%{version} export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-lws.sh ${RPM_BUILD_DIR}/%{name}-%{version} +export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-libbpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} %build # Conf step @@ -412,6 +413,7 @@ install_go install -m 0640 -p go.d.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/go.d.plugin" ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-dashboard.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_datadir}/%{name}/web +${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-ebpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d %pre diff --git a/packaging/bundle-dashboard.sh b/packaging/bundle-dashboard.sh index 8e68ff13fb..9cab084ed2 100755 --- a/packaging/bundle-dashboard.sh +++ b/packaging/bundle-dashboard.sh @@ -6,9 +6,9 @@ WEBDIR="${2}" DASHBOARD_TARBALL="dashboard.tar.gz" DASHBOARD_VERSION="$(cat "${SRCDIR}/packaging/dashboard.version")" -mkdir -p "${SRCDIR}/tmp" +mkdir -p "${SRCDIR}/tmp/dashboard" curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/dashboard/releases/download/${DASHBOARD_VERSION}/${DASHBOARD_TARBALL}" > "${DASHBOARD_TARBALL}" || exit 1 sha256sum -c "${SRCDIR}/packaging/dashboard.checksums" || exit 1 -tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp" || exit 1 +tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp/dashboard" || exit 1 # shellcheck disable=SC2046 -cp -a $(find "${SRCDIR}/tmp/build" -mindepth 1 -maxdepth 1) "${WEBDIR}" +cp -a $(find "${SRCDIR}/tmp/dashboard/build" -mindepth 1 -maxdepth 1) "${WEBDIR}" diff --git a/packaging/bundle-ebpf.sh b/packaging/bundle-ebpf.sh new file mode 100755 index 0000000000..ecc365ea44 --- /dev/null +++ b/packaging/bundle-ebpf.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +SRCDIR="${1}" +PLUGINDIR="${2}" + +EBPF_VERSION="$(cat "${SRCDIR}/packaging/ebpf.version")" +EBPF_TARBALL="netdata-kernel-collector-glibc-${EBPF_VERSION}.tar.xz" + +mkdir -p "${SRCDIR}/tmp/ebpf" +curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/kernel-collector/releases/download/${EBPF_VERSION}/${EBPF_TARBALL}" > "${EBPF_TARBALL}" || exit 1 +grep "${EBPF_TARBALL}" "${SRCDIR}/packaging/ebpf.checksums" | sha256sum -c - || exit 1 +tar -xaf "${EBPF_TARBALL}" -C "${SRCDIR}/tmp/ebpf" || exit 1 +# shellcheck disable=SC2046 +cp -a $(find "${SRCDIR}/tmp/ebpf" -mindepth 1 -maxdepth 1) "${PLUGINDIR}" diff --git a/packaging/bundle-libbpf.sh b/packaging/bundle-libbpf.sh new file mode 100755 index 0000000000..53b8da290d --- /dev/null +++ b/packaging/bundle-libbpf.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +LIBBPF_TARBALL="v$(cat "${1}/packaging/libbpf.version").tar.gz" +LIBBPF_BUILD_PATH="${1}/externaldeps/libbpf/libbpf-$(cat "${1}/packaging/libbpf.version")" + +if [ "$(uname -m)" = x86_64 ]; then + lib_subdir="lib64" +else + lib_subdir="lib" +fi + +mkdir -p "${1}/externaldeps/libbpf" || exit 1 +curl -sSL --connect-timeout 10 --retry 3 "https://github.com/libbpf/libbpf/archive/${LIBBPF_TARBALL}" > "${LIBBPF_TARBALL}" || exit 1 +sha256sum -c "${1}/packaging/libbpf.checksums" || exit 1 +tar -xzf "${LIBBPF_TARBALL}" -C "${1}/externaldeps/libbpf" || exit 1 +LPWD="${PWD}" +cd "${LIBBPF_BUILD_PATH}" || exit 1 +patch -p1 < "${1}/libnetdata/ebpf/libbpf.c.diff" || exit 1 +cd "${LPWD}" || exit 1 +make -C "${LIBBPF_BUILD_PATH}/src" BUILD_STATIC_ONLY=1 OBJDIR=build/ DESTDIR=../ install || exit 1 +cp -a "${LIBBPF_BUILD_PATH}/usr/${lib_subdir}/libbpf.a" "${1}/externaldeps/libbpf" || exit 1 +cp -a "${LIBBPF_BUILD_PATH}/usr/include" "${1}/externaldeps/libbpf" || exit 1 -- cgit v1.2.3