summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/dockerfiles/Dockerfile.build_test1
-rw-r--r--.github/workflows/build-and-install.yml82
-rw-r--r--CMakeLists.txt43
-rw-r--r--configure.ac87
-rw-r--r--exporting/aws_kinesis/README.md13
5 files changed, 173 insertions, 53 deletions
diff --git a/.github/dockerfiles/Dockerfile.build_test b/.github/dockerfiles/Dockerfile.build_test
index 5e6de6d603..1dc3e303d6 100644
--- a/.github/dockerfiles/Dockerfile.build_test
+++ b/.github/dockerfiles/Dockerfile.build_test
@@ -7,6 +7,5 @@ ENV PRE=${PRE}
COPY . /netdata
-RUN chmod +x /netdata/rmjsonc.sh
RUN /bin/sh /netdata/prep-cmd.sh
RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml
index 40eb269cdf..cf8b212e03 100644
--- a/.github/workflows/build-and-install.yml
+++ b/.github/workflows/build-and-install.yml
@@ -108,7 +108,7 @@ jobs:
RMJSONC: ${{ matrix.rmjsonc }}
run: |
echo $PRE > ./prep-cmd.sh
- echo $RMJSONC > ./rmjsonc.sh
+ echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh
docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
- name: Regular build on ${{ matrix.distro }}
run: |
@@ -124,3 +124,83 @@ jobs:
run: |
docker run -w /netdata test \
/bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
+ aws-kinesis-build:
+ name: With AWS Kinesis SDK
+ strategy:
+ fail-fast: false
+ matrix:
+ distro:
+ # XXX: CentOS 8.x is broken (libjudy)
+ # - 'centos:8'
+ - 'debian:buster'
+ - 'fedora:32'
+ - 'ubuntu:20.04'
+ include:
+ #- distro: 'centos:8'
+ # pre: >-
+ # yum -y update &&
+ # yum -y groupinstall 'Development Tools' &&
+ # yum -y install libcurl-devel openssl-devel libuuid-devel
+ # build_kinesis: >-
+ # git clone https://github.com/aws/aws-sdk-cpp.git &&
+ # cmake -DCMAKE_INSTALL_PREFIX=/usr
+ # -DBUILD_ONLY=kinesis
+ # ./aws-sdk-cpp &&
+ # make &&
+ # make install
+ - distro: 'debian:buster'
+ pre: >-
+ apt-get update &&
+ apt-get install -y build-essential &&
+ apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
+ build_kinesis: >-
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
+ -DBUILD_ONLY=kinesis
+ ./aws-sdk-cpp &&
+ make &&
+ make install
+ - distro: 'fedora:32'
+ pre: >-
+ dnf -y update &&
+ dnf -y groupinstall 'Development Tools' &&
+ dnf -y install libcurl-devel openssl-devel libuuid-devel
+ build_kinesis: >-
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
+ -DBUILD_ONLY=kinesis
+ ./aws-sdk-cpp &&
+ make &&
+ make install
+ - distro: 'ubuntu:20.04'
+ pre: >-
+ apt-get update &&
+ apt-get install -y build-essential &&
+ apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev
+ build_kinesis: >-
+ git clone https://github.com/aws/aws-sdk-cpp.git &&
+ cmake -DCMAKE_INSTALL_PREFIX=/usr
+ -DBUILD_ONLY=kinesis
+ ./aws-sdk-cpp &&
+ make &&
+ make install
+ runs-on: ubuntu-latest
+ steps:
+ - name: Git clone repository
+ uses: actions/checkout@v2
+ - name: install-required-packages.sh on ${{ matrix.distro }}
+ env:
+ PRE: ${{ matrix.pre }}
+ BUILD_KINESIS: ${{ matrix.build_kinesis }}
+ run: |
+ echo $PRE > ./prep-cmd.sh
+ echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh
+ docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
+ - name: Build on ${{ matrix.distro }}
+ env:
+ RUNCMD: >-
+ ./build-kinesis.sh &&
+ ./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis
+ run: |
+ docker run -w /netdata test \
+ /bin/sh -c "$RUNCMD"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4633b7bc9c..3278411a79 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -237,27 +237,51 @@ pkg_check_modules(CURL libcurl)
# ${CURL_CFLAGS_OTHER}
# ${CURL_INCLUDE_DIRS}
+# -----------------------------------------------------------------------------
+# Detect libaws-c-common
+
+find_library(HAVE_AWS_CHECKSUMS aws-checksums)
+# later we use:
+# ${HAVE_AWS_CHECKSUMS}
+
+# -----------------------------------------------------------------------------
+# Detect libaws-c-common
+
+find_library(HAVE_AWS_COMMON aws-c-common)
+# later we use:
+# ${HAVE_AWS_COMMON}
+
+# -----------------------------------------------------------------------------
+# Detect libaws-c-common
+
+find_library(HAVE_AWS_EVENT_STREAM aws-c-event-stream)
+# later we use:
+# ${HAVE_AWS_EVENT_STREAM}
# -----------------------------------------------------------------------------
# Detect libaws-cpp-sdk-core
-find_library(HAVE_AWS aws-cpp-sdk-core)
+pkg_check_modules(AWS_CORE aws-cpp-sdk-core)
# later we use:
-# ${HAVE_AWS}
+# ${AWS_CORE_LIBRARIES}
+# ${AWS_CORE_CFLAGS_OTHER}
+# ${AWS_CORE_INCLUDE_DIRS}
# -----------------------------------------------------------------------------
# Detect libaws-cpp-sdk-kinesis
-find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
+pkg_check_modules(KINESIS aws-cpp-sdk-kinesis)
# later we use:
-# ${HAVE_KINESIS}
+# ${KINESIS_LIBRARIES}
+# ${KINESIS_CFLAGS_OTHER}
+# ${KINESIS_INCLUDE_DIRS}
# -----------------------------------------------------------------------------
# Detect libgrpc
pkg_check_modules(GRPC grpc)
# later we use:
-# ${GRPCF_LIBRARIES}
+# ${GRPC_LIBRARIES}
# ${GRPC_CFLAGS_OTHER}
# ${GRPC_INCLUDE_DIRS}
@@ -844,7 +868,8 @@ add_definitions(
# -----------------------------------------------------------------------------
# kinesis backend
-IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
+IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND
+ CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
SET(ENABLE_BACKEND_KINESIS True)
ELSE()
SET(ENABLE_BACKEND_KINESIS False)
@@ -853,8 +878,10 @@ ENDIF()
IF(ENABLE_BACKEND_KINESIS)
message(STATUS "kinesis backend: enabled")
list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES})
- list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
- list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
+ list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES}
+ ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
+ list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS}
+ ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
ELSE()
message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")
diff --git a/configure.ac b/configure.ac
index f9379a9bfa..7d9938f8b3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1074,46 +1074,45 @@ PKG_CHECK_MODULES(
[have_libcurl=no]
)
-AC_CHECK_LIB(
+PKG_CHECK_MODULES(
+ [AWS_CPP_SDK_CORE],
[aws-cpp-sdk-core],
- [cJSON_free],
- [have_libaws_cpp_sdk_core=yes],
- [have_libaws_cpp_sdk_core=no],
- [${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
+ [AC_CHECK_LIB(
+ [aws-cpp-sdk-core],
+ [cJSON_free],
+ [have_libaws_cpp_sdk_core=yes],
+ [have_libaws_cpp_sdk_core=no]
+ )],
+ [have_libaws_cpp_sdk_core=no]
)
-AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
-
-if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
- AC_LANG_SAVE
- AC_LANG_CPLUSPLUS
- save_LIBS="${LIBS}"
- LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
- save_CXXFLAGS="${CXXFLAGS}"
- CXXFLAGS="${CXXFLAGS} -std=c++11"
+PKG_CHECK_MODULES(
+ [AWS_CPP_SDK_KINESIS],
+ [aws-cpp-sdk-kinesis],
+ [have_libaws_cpp_sdk_kinesis=yes],
+ [have_libaws_cpp_sdk_kinesis=no]
+)
- AC_TRY_LINK(
- [
- #include <aws/core/Aws.h>
- #include <aws/core/client/ClientConfiguration.h>
- #include <aws/core/auth/AWSCredentials.h>
- #include <aws/core/utils/Outcome.h>
- #include <aws/kinesis/KinesisClient.h>
- #include <aws/kinesis/model/PutRecordRequest.h>
- ],
- [Aws::Kinesis::Model::PutRecordRequest request;],
- [have_libaws_cpp_sdk_kinesis=yes],
- [have_libaws_cpp_sdk_kinesis=no]
- )
+AC_CHECK_LIB(
+ [aws-checksums],
+ [aws_checksums_crc32],
+ [have_libaws_checksums=yes],
+ [have_libaws_checksums=no]
+)
- LIBS="${save_LIBS}"
- CXXFLAGS="${save_CXXFLAGS}"
- AC_LANG_RESTORE
-else
- have_libaws_cpp_sdk_kinesis=no
-fi
+AC_CHECK_LIB(
+ [aws-c-common],
+ [aws_default_allocator],
+ [have_libaws_c_common=yes],
+ [have_libaws_c_common=no]
+)
-AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
+AC_CHECK_LIB(
+ [aws-c-event-stream],
+ [aws_event_stream_library_init],
+ [have_libaws_c_event_stream=yes],
+ [have_libaws_c_event_stream=no]
+)
test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
@@ -1130,14 +1129,26 @@ test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
AC_MSG_ERROR([libcrypto required but not found])
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_checksums}" != "yes" \
+ -a "${have_libaws_c_common}" != "yes" \
+ -a "${have_libaws_c_event_stream}" != "yes" && \
+ AC_MSG_ERROR([AWS SKD third party dependencies required but not found])
+
AC_MSG_CHECKING([if kinesis backend should be enabled])
-if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
- -a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
+if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \
+ -a "${have_libaws_cpp_sdk_core}" = "yes" \
+ -a "${have_libaws_checksums}" = "yes" \
+ -a "${have_libaws_c_common}" = "yes" \
+ -a "${have_libaws_c_event_stream}" = "yes" \
+ -a "${have_libcurl}" = "yes" \
+ -a "${have_libssl}" = "yes" \
+ -a "${have_libcrypto}" = "yes"; then
enable_backend_kinesis="yes"
AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
- CXX11FLAG="-std=c++11"
- OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
+ CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
+ OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \
+ ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
else
enable_backend_kinesis="no"
fi
diff --git a/exporting/aws_kinesis/README.md b/exporting/aws_kinesis/README.md
index 538ff34266..299fec5817 100644
--- a/exporting/aws_kinesis/README.md
+++ b/exporting/aws_kinesis/README.md
@@ -10,20 +10,23 @@ sidebar_label: AWS Kinesis Data Streams
## Prerequisites
To use AWS Kinesis for metric collecting and processing, you should first
-[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++. Netdata
-works with the SDK version 1.7.121. Other versions might work correctly as well, but they were not tested with Netdata.
+[install](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) AWS SDK for C++.
`libcrypto`, `libssl`, and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata
should be re-installed from the source. The installer will detect that the required libraries are now available.
-If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the
-building process could take a very long time. Note that the default installation path for the libraries is
+If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY=kinesis`. Otherwise, the
+build process could take a very long time. Note, that the default installation path for the libraries is
`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is
advisable to use the following options to `cmake` while building the AWS SDK:
```sh
-cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
+sudo cmake -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>
```
+The `-DCMAKE_INSTALL_PREFIX=/usr` option also ensures that
+[third party dependencies](https://github.com/aws/aws-sdk-cpp#third-party-dependencies) are installed in your system
+during the SDK build process.
+
## Configuration
To enable data sending to the Kinesis service, run `./edit-config exporting.conf` in the Netdata configuration directory
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-23 14:37:36 +0000