Add AWS Kinesis backend (#5914)

* Add Kinesis backend

* Separate config file

* Send data in chunks

* Fix minor issues

* Add error handling

* Use existing JSON functions

* Do not retry on send failure

* Implement building with autotools

* Implement building with CMake

* Fix CMake variables

* Fix build when C++ compiler is not available

* Add checks for C++11

* Don't reinitialize API

* Don't reinitialize client

* Minor cleanup

* Fix Codacy warning

* Separate sending records and receiving results

* Add documentation

* Make connection timeout configurable

* Fix operation metrics

* Fix typo

* Change parameter names for credentials

* Allow using the default SDK credentials configuration

1 files changed, 77 insertions, 2 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 46a3fed32f..72a31ec4ef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,8 +3,8 @@
 # This file is only used for development (netdata in Clion)
 # It can build netdata, but you are on your own...
 
-cmake_minimum_required(VERSION 3.0.2)
-project(netdata C)
+cmake_minimum_required(VERSION 3.1.0)
+project(netdata C CXX)
 
 find_package(Threads REQUIRED)
 find_package(PkgConfig REQUIRED)
@@ -158,6 +158,50 @@ ENDIF(MACOS)
 
 
 # -----------------------------------------------------------------------------
+# Detect libcrypto
+
+pkg_check_modules(CRYPTO libcrypto)
+# later we use:
+# ${CRYPTO_LIBRARIES}
+# ${CRYPTO_CFLAGS_OTHER}
+# ${CRYPTO_INCLUDE_DIRS}
+
+
+# -----------------------------------------------------------------------------
+# Detect libssl
+
+pkg_check_modules(SSL libssl)
+# later we use:
+# ${SSL_LIBRARIES}
+# ${SSL_CFLAGS_OTHER}
+# ${SSL_INCLUDE_DIRS}
+
+
+# -----------------------------------------------------------------------------
+# Detect libcurl
+
+pkg_check_modules(CURL libcurl)
+# later we use:
+# ${CURL_LIBRARIES}
+# ${CURL_CFLAGS_OTHER}
+# ${CURL_INCLUDE_DIRS}
+
+
+# -----------------------------------------------------------------------------
+# Detect libaws-cpp-sdk-core
+
+find_library(HAVE_AWS aws-cpp-sdk-core)
+# later we use:
+# ${HAVE_AWS}
+
+# -----------------------------------------------------------------------------
+# Detect libaws-cpp-sdk-kinesis
+
+find_library(HAVE_KINESIS aws-cpp-sdk-kinesis)
+# later we use:
+# ${HAVE_KINESIS}
+
+# -----------------------------------------------------------------------------
 # netdata files
 
 set(LIBNETDATA_FILES
@@ -440,6 +484,13 @@ set(BACKENDS_PLUGIN_FILES
         backends/prometheus/backend_prometheus.h
         )
 
+set(KINESIS_BACKEND_FILES
+        backends/aws_kinesis/aws_kinesis.c
+        backends/aws_kinesis/aws_kinesis.h
+        backends/aws_kinesis/aws_kinesis_put_record.cc
+        backends/aws_kinesis/aws_kinesis_put_record.h
+        )
+
 set(DAEMON_FILES
         daemon/common.c
         daemon/common.h
@@ -486,6 +537,25 @@ add_definitions(
 )
 
 # -----------------------------------------------------------------------------
+# kinesis backend
+
+IF(HAVE_KINESIS AND HAVE_AWS AND CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)
+    SET(ENABLE_BACKEND_KINESIS True)
+ELSE()
+    SET(ENABLE_BACKEND_KINESIS False)
+ENDIF()
+
+IF(ENABLE_BACKEND_KINESIS)
+    message(STATUS "kinesis backend: enabled")
+    list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES})
+    list(APPEND NETDATA_COMMON_LIBRARIES aws-cpp-sdk-kinesis aws-cpp-sdk-core ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})
+    list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
+    list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})
+ELSE()
+    message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")
+ENDIF()
+
+# -----------------------------------------------------------------------------
 # netdata
 
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m ${CMAKE_THREAD_LIBS_INIT})
@@ -522,6 +592,11 @@ ELSEIF(MACOS)
 
 ENDIF()
 
+IF(ENABLE_BACKEND_KINESIS)
+    set_property(TARGET netdata PROPERTY CXX_STANDARD 11)
+    set_property(TARGET netdata PROPERTY CMAKE_CXX_STANDARD_REQUIRED ON)
+ENDIF()
+
 IF(IPMI_LIBRARIES)
     SET(ENABLE_PLUGIN_FREEIPMI True)
 ELSE()