diff options
author | Austin S. Hemmelgarn <austin@netdata.cloud> | 2024-01-08 08:40:41 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-08 08:40:41 -0500 |
commit | 2710a3134d33aa02d14fd08d0446b187e5527662 (patch) | |
tree | 2fcb48af644e5f19e10e8c6e33a61547c557a128 | |
parent | 9638fddbdd66a2f65b5c5f069b57c4a5fa54a7ab (diff) |
Fix (and improve) Coverity scanning. (#16605)
* Split feature detection code to functions.
This lets us reuse it in places like the coverity scan script.
* Add ninja to the coverity build environment.
* Restructure coverity scan script to use cmake.
* Fix incorrect function name.
* Add extra plugin deps for coverity scan environment.
* Fix new issues introduced in coverity script.
* Fix things broken by previous fix.
* Fix Docker image builds.
* Assorted fixes.
* Correctly handle common case of set but empty prefix.
-rw-r--r-- | .github/workflows/coverity.yml | 3 | ||||
-rwxr-xr-x | coverity-scan.sh | 40 | ||||
-rwxr-xr-x | netdata-installer.sh | 107 | ||||
-rw-r--r-- | packaging/docker/Dockerfile | 2 | ||||
-rw-r--r-- | packaging/installer/functions.sh | 106 |
5 files changed, 127 insertions, 131 deletions
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index eb68c302b2..4f5010a5c6 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -33,7 +33,8 @@ jobs: sudo apt-get install -y libjson-c-dev libyaml-dev libipmimonitoring-dev \ libcups2-dev libsnappy-dev libprotobuf-dev \ libprotoc-dev libssl-dev protobuf-compiler \ - libnetfilter-acct-dev + libnetfilter-acct-dev libmongoc-dev libxen-dev \ + libsystemd-dev ninja-build - name: Run coverity-scan id: run env: diff --git a/coverity-scan.sh b/coverity-scan.sh index 8466b21a7f..ce2c86342f 100755 --- a/coverity-scan.sh +++ b/coverity-scan.sh @@ -42,11 +42,19 @@ INSTALL_DIR="/opt" # the version of coverity to use COVERITY_BUILD_VERSION="${COVERITY_BUILD_VERSION:-cov-analysis-linux64-2023.6.2}" -# TODO: For some reasons this does not fully load on Debian 10 (Haven't checked if it happens on other distros yet), it breaks -source packaging/installer/functions.sh || echo "Failed to fully load the functions library" +. packaging/installer/functions.sh -cpus=$(find_processors) -[ -z "${cpus}" ] && cpus=1 +JOBS=$(find_processors) +[ -z "${JOBS}" ] && JOBS=1 + +if command -v ninja 2>&1; then + ninja="$(command -v ninja)" +fi + +CMAKE_OPTS="${ninja:+-G Ninja}" +BUILD_OPTS="VERBOSE=1" +[ -n "${ninja}" ] && BUILD_OPTS="-v" +NETDATA_BUILD_DIR="${NETDATA_BUILD_DIR:-./cmake-build-release/}" if [ -f ".coverity-scan.conf" ]; then source ".coverity-scan.conf" @@ -102,19 +110,20 @@ scanit() { progress "Working on netdata version: ${version}" progress "Cleaning up old builds..." - run make clean || echo >&2 "Nothing to clean" + rm -rf "${NETDATA_BUILD_DIR}" [ -d "cov-int" ] && rm -rf "cov-int" [ -f netdata-coverity-analysis.tgz ] && run rm netdata-coverity-analysis.tgz progress "Configuring netdata source..." + USE_SYSTEM_PROTOBUF=1 + prepare_cmake_options - run autoreconf -ivf - run ./configure ${OTHER_OPTIONS} + run cmake ${NETDATA_CMAKE_OPTIONS} progress "Analyzing netdata..." - run "${covbuild}" --dir cov-int make -j${cpus} + run "${covbuild}" --dir cov-int cmake --build "${NETDATA_BUILD_DIR}" --parallel ${JOBS} -- ${BUILD_OPTS} echo >&2 "Compressing analysis..." run tar czvf netdata-coverity-analysis.tgz cov-int @@ -170,21 +179,6 @@ installit() { return 0 } -OTHER_OPTIONS="--disable-lto" -OTHER_OPTIONS+=" --with-zlib" -OTHER_OPTIONS+=" --with-math" -OTHER_OPTIONS+=" --enable-lz4" -OTHER_OPTIONS+=" --enable-openssl" -OTHER_OPTIONS+=" --enable-jsonc" -OTHER_OPTIONS+=" --enable-plugin-nfacct" -OTHER_OPTIONS+=" --enable-plugin-freeipmi" -OTHER_OPTIONS+=" --enable-plugin-cups" -OTHER_OPTIONS+=" --enable-exporting-prometheus-remote-write" -# TODO: enable these plugins too -#OTHER_OPTIONS+=" --enable-plugin-xenstat" -#OTHER_OPTIONS+=" --enable-exporting-kinesis" -#OTHER_OPTIONS+=" --enable-exporting-mongodb" - FOUND_OPTS="NO" while [ -n "${1}" ]; do if [ "${1}" = "--with-install" ]; then diff --git a/netdata-installer.sh b/netdata-installer.sh index 717f526afe..5fb8cf9741 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -454,15 +454,6 @@ if [ "$(uname -s)" = "Linux" ] && [ -f /proc/meminfo ]; then fi fi -enable_feature() { - NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed -e "s/-DENABLE_${1}=Off[[:space:]]*//g" -e "s/-DENABLE_${1}=On[[:space:]]*//g")" - if [ "${2}" -eq 1 ]; then - NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=On/")" - else - NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=Off/")" - fi -} - # set default make options if [ -z "${MAKEOPTS}" ]; then MAKEOPTS="-j$(find_processors)" @@ -1104,35 +1095,6 @@ echo >&2 NETDATA_BUILD_DIR="${NETDATA_BUILD_DIR:-./cmake-build-release/}" rm -rf "${NETDATA_BUILD_DIR}" -check_for_module() { - if [ -z "${pkgconf}" ]; then - pkgconf="$(command -v pkgconf 2>/dev/null)" - [ -z "${pkgconf}" ] && pkgconf="$(command -v pkg-config 2>/dev/null)" - [ -z "${pkgconf}" ] && fatal "Unable to find a usable pkgconf/pkg-config command, cannot build Netdata." I0013 - fi - - "${pkgconf}" "${1}" - return "${?}" -} - -check_for_feature() { - feature_name="${1}" - feature_state="${2}" - shift 2 - feature_modules="${*}" - - if [ -z "${feature_state}" ]; then - # shellcheck disable=SC2086 - if check_for_module ${feature_modules}; then - enable_feature "${feature_name}" 1 - else - enable_feature "${feature_name}" 0 - fi - else - enable_feature "${feature_name}" "${feature_state}" - fi -} - # function to extract values from the config file config_option() { section="${1}" @@ -1161,74 +1123,7 @@ NETDATA_GROUP="$(id -g -n "${NETDATA_USER}" 2> /dev/null)" [ -z "${NETDATA_GROUP}" ] && NETDATA_GROUP="${NETDATA_USER}" echo >&2 "Netdata user and group set to: ${NETDATA_USER}/${NETDATA_GROUP}" -NETDATA_CMAKE_OPTIONS="-S ./ -B ${NETDATA_BUILD_DIR} ${CMAKE_OPTS} -DCMAKE_INSTALL_PREFIX=${NETDATA_PREFIX} ${NETDATA_USER:+-DNETDATA_USER=${NETDATA_USER}} ${NETDATA_CMAKE_OPTIONS} " - -# Feature autodetection code starts here - -if [ "${USE_SYSTEM_PROTOBUF}" -eq 1 ]; then - enable_feature BUNDLED_PROTOBUF 0 -else - enable_feature BUNDLED_PROTOBUF 1 -fi - -if [ -z "${ENABLE_SYSTEMD_PLUGIN}" ]; then - if check_for_module libsystemd; then - if check_for_module libelogind; then - ENABLE_SYSTEMD_JOURNAL=0 - else - ENABLE_SYSTEMD_JOURNAL=1 - fi - else - ENABLE_SYSTEMD_JOURNAL=0 - fi -fi - -enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}" - -[ -z "${NETDATA_ENABLE_ML}" ] && NETDATA_ENABLE_ML=1 -enable_feature ML "${NETDATA_ENABLE_ML}" - -if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then - ENABLE_CUPS=1 -else - ENABLE_CUPS=0 -fi - -enable_feature PLUGIN_CUPS "${ENABLE_CUPS}" - -IS_LINUX=0 -[ "$(uname -s)" = "Linux" ] && IS_LINUX=1 -enable_feature PLUGIN_DEBUGFS "${IS_LINUX}" -enable_feature PLUGIN_PERF "${IS_LINUX}" -enable_feature PLUGIN_SLABINFO "${IS_LINUX}" -enable_feature PLUGIN_CGROUP_NETWORK "${IS_LINUX}" -enable_feature PLUGIN_LOCAL_LISTENERS "${IS_LINUX}" -enable_feature PLUGIN_LOGS_MANAGEMENT "${ENABLE_LOGS_MANAGEMENT}" -enable_feature LOGS_MANAGEMENT_TESTS "${ENABLE_LOGS_MANAGEMENT_TESTS}" - -enable_feature ACLK "${ENABLE_CLOUD}" -enable_feature CLOUD "${ENABLE_CLOUD}" -enable_feature BUNDLED_JSONC "${NETDATA_BUILD_JSON_C}" -enable_feature BUNDLED_YAML "${BUNDLE_YAML}" -enable_feature DBENGINE "${ENABLE_DBENGINE}" -enable_feature H2O "${ENABLE_H2O}" -enable_feature PLUGIN_EBPF "${ENABLE_EBPF}" - -ENABLE_APPS=0 - -if [ "${IS_LINUX}" = 1 ] || [ "$(uname -s)" = "FreeBSD" ]; then - ENABLE_APPS=1 -fi - -enable_feature PLUGIN_APPS "${ENABLE_APPS}" - -check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy -check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0 -check_for_feature PLUGIN_FREEIPMI "${ENABLE_FREEIPMI}" libipmimonitoring -check_for_feature PLUGIN_NFACCT "${ENABLE_NFACCT}" libnetfilter_acct libnml -check_for_feature PLUGIN_XENSTAT "${ENABLE_XENSTAT}" xenstat xenlight - -# End of feature autodetection code +prepare_cmake_options if [ -n "${NETDATA_PREPARE_ONLY}" ]; then progress "Exiting before building Netdata as requested." diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index f7a8dd43f0..f659dfab66 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -29,7 +29,7 @@ RUN chmod +x netdata-installer.sh && \ cp -rp /deps/* /usr/local/ && \ /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='$(uname -m)'" > ./system/.install-type && \ CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it --use-system-protobuf \ - ${EXTRA_INSTALL_OPTS} --disable-ebpf --one-time-build --enable-lto "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)" + ${EXTRA_INSTALL_OPTS} --disable-ebpf --install-no-prefix / "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)" # files to one directory RUN mkdir -p /app/usr/sbin/ \ diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh index 6cf8cdf145..8b30fc82ad 100644 --- a/packaging/installer/functions.sh +++ b/packaging/installer/functions.sh @@ -192,6 +192,112 @@ netdata_banner() { } # ----------------------------------------------------------------------------- +# Feature management and configuration commands + +enable_feature() { + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed -e "s/-DENABLE_${1}=Off[[:space:]]*//g" -e "s/-DENABLE_${1}=On[[:space:]]*//g")" + if [ "${2}" -eq 1 ]; then + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=On/")" + else + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=Off/")" + fi +} + +check_for_module() { + if [ -z "${pkgconf}" ]; then + pkgconf="$(command -v pkgconf 2>/dev/null)" + [ -z "${pkgconf}" ] && pkgconf="$(command -v pkg-config 2>/dev/null)" + [ -z "${pkgconf}" ] && fatal "Unable to find a usable pkgconf/pkg-config command, cannot build Netdata." I0013 + fi + + "${pkgconf}" "${1}" + return "${?}" +} + +check_for_feature() { + feature_name="${1}" + feature_state="${2}" + shift 2 + feature_modules="${*}" + + if [ -z "${feature_state}" ]; then + # shellcheck disable=SC2086 + if check_for_module ${feature_modules}; then + enable_feature "${feature_name}" 1 + else + enable_feature "${feature_name}" 0 + fi + else + enable_feature "${feature_name}" "${feature_state}" + fi +} + +prepare_cmake_options() { + NETDATA_CMAKE_OPTIONS="-S ./ -B ${NETDATA_BUILD_DIR} ${CMAKE_OPTS} ${NETDATA_PREFIX+-DCMAKE_INSTALL_PREFIX="${NETDATA_PREFIX}"} ${NETDATA_USER:+-DNETDATA_USER=${NETDATA_USER}} ${NETDATA_CMAKE_OPTIONS} " + + if [ "${USE_SYSTEM_PROTOBUF:-1}" -eq 1 ]; then + enable_feature BUNDLED_PROTOBUF 0 + else + enable_feature BUNDLED_PROTOBUF 1 + fi + + if [ -z "${ENABLE_SYSTEMD_JOURNAL}" ]; then + if check_for_module libsystemd; then + if check_for_module libelogind; then + ENABLE_SYSTEMD_JOURNAL=0 + else + ENABLE_SYSTEMD_JOURNAL=1 + fi + else + ENABLE_SYSTEMD_JOURNAL=0 + fi + fi + + enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}" + + if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then + ENABLE_CUPS=1 + else + ENABLE_CUPS=0 + fi + + enable_feature PLUGIN_CUPS "${ENABLE_CUPS}" + + IS_LINUX=0 + [ "$(uname -s)" = "Linux" ] && IS_LINUX=1 + enable_feature PLUGIN_DEBUGFS "${IS_LINUX}" + enable_feature PLUGIN_PERF "${IS_LINUX}" + enable_feature PLUGIN_SLABINFO "${IS_LINUX}" + enable_feature PLUGIN_CGROUP_NETWORK "${IS_LINUX}" + enable_feature PLUGIN_LOCAL_LISTENERS "${IS_LINUX}" + enable_feature PLUGIN_EBPF "${ENABLE_EBPF:-0}" + enable_feature PLUGIN_LOGS_MANAGEMENT "${ENABLE_LOGS_MANAGEMENT:-0}" + enable_feature LOGS_MANAGEMENT_TESTS "${ENABLE_LOGS_MANAGEMENT_TESTS:-0}" + + enable_feature ACLK "${ENABLE_CLOUD:-1}" + enable_feature CLOUD "${ENABLE_CLOUD:-1}" + enable_feature BUNDLED_JSONC "${NETDATA_BUILD_JSON_C:-0}" + enable_feature BUNDLED_YAML "${BUNDLE_YAML:-0}" + enable_feature DBENGINE "${ENABLE_DBENGINE:-1}" + enable_feature H2O "${ENABLE_H2O:-1}" + enable_feature ML "${NETDATA_ENABLE_ML:-1}" + + ENABLE_APPS=0 + + if [ "${IS_LINUX}" = 1 ] || [ "$(uname -s)" = "FreeBSD" ]; then + ENABLE_APPS=1 + fi + + enable_feature PLUGIN_APPS "${ENABLE_APPS}" + + check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy + check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0 + check_for_feature PLUGIN_FREEIPMI "${ENABLE_FREEIPMI}" libipmimonitoring + check_for_feature PLUGIN_NFACCT "${ENABLE_NFACCT}" libnetfilter_acct libnml + check_for_feature PLUGIN_XENSTAT "${ENABLE_XENSTAT}" xenstat xenlight +} + +# ----------------------------------------------------------------------------- # portable service command service_cmd="$(command -v service 2> /dev/null || true)" |