Add AWS Kinesis backend (#5914)

* Add Kinesis backend

* Separate config file

* Send data in chunks

* Fix minor issues

* Add error handling

* Use existing JSON functions

* Do not retry on send failure

* Implement building with autotools

* Implement building with CMake

* Fix CMake variables

* Fix build when C++ compiler is not available

* Add checks for C++11

* Don't reinitialize API

* Don't reinitialize client

* Minor cleanup

* Fix Codacy warning

* Separate sending records and receiving results

* Add documentation

* Make connection timeout configurable

* Fix operation metrics

* Fix typo

* Change parameter names for credentials

* Allow using the default SDK credentials configuration

1 files changed, 135 insertions, 12 deletions

diff --git a/configure.ac b/configure.ac
index 565f05c133..95b45dd42a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -37,6 +37,7 @@ m4_ifdef([AM_SILENT_RULES], [
 AC_CANONICAL_HOST
 AC_PROG_CC
 AM_PROG_CC_C_O
+AC_PROG_CXX
 AC_PROG_INSTALL
 PKG_PROG_PKG_CONFIG
 AC_USE_SYSTEM_EXTENSIONS
@@ -69,6 +70,12 @@ AC_ARG_ENABLE(
     ,
     [enable_plugin_xenstat="detect"]
 )
+AC_ARG_ENABLE(
+    [backend-kinesis],
+    [AS_HELP_STRING([--enable-backend-xenstat], [enable kinesis backend @<:@default autodetect@:>@])],
+    ,
+    [enable_backend_kinesis="detect"]
+)
 
 AC_ARG_ENABLE(
     [pedantic],
@@ -250,7 +257,7 @@ AC_MSG_CHECKING([if zlib should be used])
 if test "${with_zlib}" != "no" -a "${have_zlib}" = "yes"; then
     with_zlib="yes"
     AC_DEFINE([NETDATA_WITH_ZLIB], [1], [zlib usability])
-    OPTIONAL_ZLIB_CLFAGS="${ZLIB_CFLAGS}"
+    OPTIONAL_ZLIB_CFLAGS="${ZLIB_CFLAGS}"
     OPTIONAL_ZLIB_LIBS="${ZLIB_LIBS}"
 else
     with_zlib="no"
@@ -268,7 +275,7 @@ PKG_CHECK_MODULES(
     [AC_MSG_ERROR([libuuid required but not found. Try installing 'uuid-dev' or 'libuuid-devel'.])]
 )
 AC_DEFINE([NETDATA_WITH_UUID], [1], [uuid usability])
-OPTIONAL_UUID_CLFAGS="${UUID_CFLAGS}"
+OPTIONAL_UUID_CFLAGS="${UUID_CFLAGS}"
 OPTIONAL_UUID_LIBS="${UUID_LIBS}"
 
 
@@ -342,7 +349,7 @@ AC_MSG_CHECKING([if libcap should be used])
 if test "${with_libcap}" != "no" -a "${have_libcap}" = "yes"; then
     with_libcap="yes"
     AC_DEFINE([HAVE_CAPABILITY], [1], [libcap usability])
-    OPTIONAL_LIBCAP_CLFAGS="${LIBCAP_CFLAGS}"
+    OPTIONAL_LIBCAP_CFLAGS="${LIBCAP_CFLAGS}"
     OPTIONAL_LIBCAP_LIBS="${LIBCAP_LIBS}"
 else
     with_libcap="no"
@@ -403,7 +410,7 @@ AC_MSG_CHECKING([if freeipmi.plugin should be enabled])
 if test "${enable_plugin_freeipmi}" != "no" -a "${have_ipmimonitoring}" = "yes"; then
     enable_plugin_freeipmi="yes"
     AC_DEFINE([HAVE_FREEIPMI], [1], [ipmimonitoring usability])
-    OPTIONAL_IPMIMONITORING_CLFAGS="${IPMIMONITORING_CFLAGS}"
+    OPTIONAL_IPMIMONITORING_CFLAGS="${IPMIMONITORING_CFLAGS}"
     OPTIONAL_IPMIMONITORING_LIBS="${IPMIMONITORING_LIBS}"
 else
     enable_plugin_freeipmi="no"
@@ -451,7 +458,7 @@ if test "${enable_plugin_cups}" != "no" -a "${have_cups}" = "yes"; then
     CUPS_CFLAGS="${CUPS_CFLAGS} `$CUPSCONFIG --cflags`"
     CUPS_LIBS="${CUPS_LIBS} `$CUPSCONFIG --libs`"
 
-    OPTIONAL_CUPS_CLFAGS="${CUPS_CFLAGS}"
+    OPTIONAL_CUPS_CFLAGS="${CUPS_CFLAGS}"
     OPTIONAL_CUPS_LIBS="${CUPS_LIBS}"
 else
     enable_plugin_cups="no"
@@ -500,7 +507,7 @@ if test "${enable_plugin_nfacct}" != "no" -a "${have_libnetfilter_acct}" = "yes"
     enable_plugin_nfacct="yes"
     AC_DEFINE([HAVE_LIBMNL], [1], [libmnl usability])
     AC_DEFINE([HAVE_LIBNETFILTER_ACCT], [1], [libnetfilter_acct usability])
-    OPTIONAL_NFACCT_CLFAGS="${NFACCT_CFLAGS} ${LIBMNL_CFLAGS}"
+    OPTIONAL_NFACCT_CFLAGS="${NFACCT_CFLAGS} ${LIBMNL_CFLAGS}"
     OPTIONAL_NFACCT_LIBS="${NFACCT_LIBS} ${LIBMNL_LIBS}"
 else
     enable_plugin_nfacct="no"
@@ -567,7 +574,7 @@ if test "${enable_plugin_xenstat}" != "no" -a "${have_libxenstat}" = "yes" -a "$
     AC_DEFINE([HAVE_LIBXENSTAT], [1], [libxenstat usability])
     AC_DEFINE([HAVE_LIBXENLIGHT], [1], [libxenlight usability])
     AC_DEFINE([HAVE_LIBYAJL], [1], [libyajl usability])
-    OPTIONAL_XENSTAT_CLFAGS="${XENLIGHT_CFLAGS} ${YAJL_CFLAGS}"
+    OPTIONAL_XENSTAT_CFLAGS="${XENLIGHT_CFLAGS} ${YAJL_CFLAGS}"
     OPTIONAL_XENSTAT_LIBS="-lxenstat ${XENLIGHT_LIBS} ${YAJL_LIBS}"
 else
     enable_plugin_xenstat="no"
@@ -577,6 +584,117 @@ AM_CONDITIONAL([ENABLE_PLUGIN_XENSTAT], [test "${enable_plugin_xenstat}" = "yes"
 
 
 # -----------------------------------------------------------------------------
+# AWS Kinesis backend - libaws-cpp-sdk-kinesis, libaws-cpp-sdk-core, libssl, libcrypto, libcurl
+
+PKG_CHECK_MODULES(
+    [LIBCRYPTO],
+    [libcrypto],
+    [AC_CHECK_LIB(
+        [crypto],
+        [CRYPTO_new_ex_data],
+        [have_libcrypto=yes],
+        [have_libcrypto=no]
+    )],
+    [have_libcrypto=no]
+)
+
+PKG_CHECK_MODULES(
+    [LIBSSL],
+    [libssl],
+    [AC_CHECK_LIB(
+        [ssl],
+        [SSL_connect],
+        [have_libssl=yes],
+        [have_libssl=no]
+    )],
+    [have_libssl=no]
+)
+
+PKG_CHECK_MODULES(
+    [LIBCURL],
+    [libcurl],
+    [AC_CHECK_LIB(
+        [curl],
+        [curl_easy_init],
+        [have_libcurl=yes],
+        [have_libcurl=no]
+    )],
+    [have_libcurl=no]
+)
+
+AC_CHECK_LIB(
+    [aws-cpp-sdk-core],
+    [cJSON_free],
+    [have_libaws_cpp_sdk_core=yes],
+    [have_libaws_cpp_sdk_core=no],
+    [${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}]
+)
+
+AC_MSG_CHECKING([for Aws::Kinesis::Model::PutRecordRequest in -laws-cpp-sdk-kinesis])
+
+if test "${have_libaws_cpp_sdk_core}" = "yes" -a "${have_libcrypto}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcurl}" = "yes"; then
+    AC_LANG_SAVE
+    AC_LANG_CPLUSPLUS
+    save_LIBS="${LIBS}"
+    LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
+    save_CXXFLAGS="${CXXFLAGS}"
+    CXXFLAGS="${CXXFLAGS} -std=c++11"
+
+    AC_TRY_LINK(
+    [
+    #include <aws/core/Aws.h>
+    #include <aws/core/client/ClientConfiguration.h>
+    #include <aws/core/auth/AWSCredentials.h>
+    #include <aws/core/utils/Outcome.h>
+    #include <aws/kinesis/KinesisClient.h>
+    #include <aws/kinesis/model/PutRecordRequest.h>
+    ],
+    [Aws::Kinesis::Model::PutRecordRequest request;],
+    [have_libaws_cpp_sdk_kinesis=yes],
+    [have_libaws_cpp_sdk_kinesis=no]
+    )
+
+    LIBS="${save_LIBS}"
+    CXXFLAGS="${save_CXXFLAGS}"
+    AC_LANG_RESTORE
+else
+    have_libaws_cpp_sdk_kinesis=no
+fi
+
+AC_MSG_RESULT([${have_libaws_cpp_sdk_kinesis}])
+
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \
+    AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])
+
+test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" != "yes" && \
+    AC_MSG_ERROR([libaws-cpp-sdk-core required but not found. try installing AWS C++ SDK])
+
+test "${enable_backend_kinesis}" = "yes" -a "${have_libcurl}" != "yes" && \
+    AC_MSG_ERROR([libcurl required but not found])
+
+test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \
+    AC_MSG_ERROR([libssl required but not found])
+
+test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \
+    AC_MSG_ERROR([libcrypto required but not found])
+
+AC_MSG_CHECKING([if kinesis backend should be enabled])
+if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" = "yes" \
+                                            -a "${have_libcurl}" = "yes" -a "${have_libssl}" = "yes" -a "${have_libcrypto}" = "yes"; then
+    enable_backend_kinesis="yes"
+    AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
+    OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
+    CXX11FLAG="-std=c++11"
+    OPTIONAL_KINESIS_LIBS="-laws-cpp-sdk-kinesis -laws-cpp-sdk-core ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
+else
+    enable_backend_kinesis="no"
+fi
+
+AC_MSG_RESULT([${enable_backend_kinesis}])
+AM_CONDITIONAL([ENABLE_BACKEND_KINESIS], [test "${enable_backend_kinesis}" = "yes"])
+
+
+# -----------------------------------------------------------------------------
 # check for setns() - cgroup-network
 
 AC_CHECK_FUNC([setns])
@@ -600,7 +718,7 @@ if test "${enable_lto}" != "no"; then
 fi
 if test "${have_lto}" = "yes"; then
     oCFLAGS="${CFLAGS}"
-    CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CLFAGS} ${OPTIONAL_NFACCT_CLFAGS} ${OPTIONAL_ZLIB_CLFAGS} ${OPTIONAL_UUID_CLFAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CLFAGS} ${OPTIONAL_XENSTAT_FLAGS}"
+    CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} ${OPTIONAL_KINESIS_CFLAGS}"
     ac_cv_c_lto_cross_compile="${enable_lto}"
     test "${ac_cv_c_lto_cross_compile}" != "yes" && ac_cv_c_lto_cross_compile="no"
     AC_C_LTO
@@ -647,6 +765,8 @@ AC_SUBST([logdir])
 AC_SUBST([pluginsdir])
 AC_SUBST([webdir])
 
+CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
+
 CPPFLAGS="\
     -DTARGET_OS=${build_target_id} \
 	-DVARLIB_DIR=\"\\\"${varlibdir}\\\"\" \
@@ -659,13 +779,13 @@ CPPFLAGS="\
 	-DWEB_DIR=\"\\\"${webdir}\\\"\" \
 "
 
-AC_SUBST([OPTIONAL_MATH_CLFAGS])
+AC_SUBST([OPTIONAL_MATH_CFLAGS])
 AC_SUBST([OPTIONAL_MATH_LIBS])
-AC_SUBST([OPTIONAL_NFACCT_CLFAGS])
+AC_SUBST([OPTIONAL_NFACCT_CFLAGS])
 AC_SUBST([OPTIONAL_NFACCT_LIBS])
-AC_SUBST([OPTIONAL_ZLIB_CLFAGS])
+AC_SUBST([OPTIONAL_ZLIB_CFLAGS])
 AC_SUBST([OPTIONAL_ZLIB_LIBS])
-AC_SUBST([OPTIONAL_UUID_CLFAGS])
+AC_SUBST([OPTIONAL_UUID_CFLAGS])
 AC_SUBST([OPTIONAL_UUID_LIBS])
 AC_SUBST([OPTIONAL_LIBCAP_CFLAGS])
 AC_SUBST([OPTIONAL_LIBCAP_LIBS])
@@ -675,6 +795,8 @@ AC_SUBST([OPTIONAL_CUPS_CFLAGS])
 AC_SUBST([OPTIONAL_CUPS_LIBS])
 AC_SUBST([OPTIONAL_XENSTAT_CFLAGS])
 AC_SUBST([OPTIONAL_XENSTAT_LIBS])
+AC_SUBST([OPTIONAL_KINESIS_CFLAGS])
+AC_SUBST([OPTIONAL_KINESIS_LIBS])
 
 
 AC_CONFIG_FILES([
@@ -685,6 +807,7 @@ AC_CONFIG_FILES([
     backends/Makefile
     backends/opentsdb/Makefile
     backends/prometheus/Makefile
+    backends/aws_kinesis/Makefile
     collectors/Makefile
     collectors/apps.plugin/Makefile
     collectors/cgroups.plugin/Makefile