Prometheus remote write backend (#6062)

* Add Prometheus remote write backend prototype

* Fix autotools issues

* Send HTTP POST request

* Add parameters to HTTP header

* Discard HTTP responce 200

* Update CMake build configuration

* Fix Codacy issue

* Check for C++ binary

* Fix compilation without remote write backend

* Add options to the installer script

* Fix configure script warning

* Fix make dist

* Downgrade to ByteSize for better compatibility

* Integrate remote write more tightly into the existing backends code

* Cleanup

* Fix build error

* Parse host tags

* Fix Codacy issue

* Fix counters for buffered data

* Rename preprocessor symbol

* Better error handling

* Cleanup

* Update the documentation

1 files changed, 71 insertions, 2 deletions

diff --git a/configure.ac b/configure.ac
index 59bab84353..7c784c8f99 100644
--- a/configure.ac
+++ b/configure.ac
@@ -76,7 +76,12 @@ AC_ARG_ENABLE(
     ,
     [enable_backend_kinesis="detect"]
 )
-
+AC_ARG_ENABLE(
+    [backend-prometheus-remote-write],
+    [AS_HELP_STRING([--enable-backend-prometheus-remote-write], [enable prometheus remote write backend @<:@default autodetect@:>@])],
+    ,
+    [enable_backend_prometheus_remote_write="detect"]
+)
 AC_ARG_ENABLE(
     [pedantic],
     [AS_HELP_STRING([--enable-pedantic], [enable pedantic compiler warnings @<:@default disabled@:>@])],
@@ -792,6 +797,65 @@ AM_CONDITIONAL([ENABLE_BACKEND_KINESIS], [test "${enable_backend_kinesis}" = "ye
 
 
 # -----------------------------------------------------------------------------
+# Prometheus remote write backend - libprotobuf, libsnappy, protoc
+
+PKG_CHECK_MODULES(
+    [PROTOBUF],
+    [protobuf],
+    [have_libprotobuf=yes],
+    [have_libprotobuf=no]
+)
+
+PKG_CHECK_MODULES(
+    [SNAPPY],
+    [snappy],
+    [have_libsnappy=yes],
+    [have_libsnappy=no]
+)
+
+AC_PATH_PROG([PROTOC], [protoc], [no])
+AS_IF(
+    [test x"${PROTOC}" == x"no"],
+    [have_protoc=no],
+    [have_protoc=yes]
+)
+
+AC_PATH_PROG([CXX_BINARY], [${CXX}], [no])
+AS_IF(
+    [test x"${CXX_BINARY}" == x"no"],
+    [have_CXX_compiler=no],
+    [have_CXX_compiler=yes]
+)
+
+test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_libprotobuf}" != "yes" && \
+    AC_MSG_ERROR([libprotobuf required but not found. try installing protobuf])
+
+test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_libsnappy}" != "yes" && \
+    AC_MSG_ERROR([libsnappy required but not found. try installing snappy])
+
+test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_protoc}" != "yes" && \
+    AC_MSG_ERROR([protoc compiler required but not found. try installing protobuf])
+
+test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_CXX_compiler}" != "yes" && \
+    AC_MSG_ERROR([C++ compiler required but not found. try installing g++])
+
+AC_MSG_CHECKING([if prometheus remote write backend should be enabled])
+if test "${enable_backend_prometeus_remote_write}" != "no" -a "${have_libprotobuf}" = "yes" -a "${have_libsnappy}" = "yes" \
+                                                           -a "${have_protoc}" = "yes" -a "${have_CXX_compiler}" = "yes"; then
+    enable_backend_prometheus_remote_write="yes"
+    AC_DEFINE([ENABLE_PROMETHEUS_REMOTE_WRITE], [1], [Prometheus remote write API usability])
+    OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS="${PROTOBUF_CFLAGS} ${SNAPPY_CFLAGS}"
+    CXX11FLAG="-std=c++11"
+    OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS="${PROTOBUF_LIBS} ${SNAPPY_LIBS} "
+else
+    enable_backend_prometheus_remote_write="no"
+fi
+
+AC_MSG_RESULT([${enable_backend_prometheus_remote_write}])
+AM_CONDITIONAL([ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE], [test "${enable_backend_prometheus_remote_write}" = "yes"])
+
+
+# -----------------------------------------------------------------------------
 # check for setns() - cgroup-network
 
 AC_CHECK_FUNC([setns])
@@ -815,7 +879,7 @@ if test "${enable_lto}" != "no"; then
 fi
 if test "${have_lto}" = "yes"; then
     oCFLAGS="${CFLAGS}"
-    CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} ${OPTIONAL_KINESIS_CFLAGS}"
+    CFLAGS="${CFLAGS} -flto ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS}"
     ac_cv_c_lto_cross_compile="${enable_lto}"
     test "${ac_cv_c_lto_cross_compile}" != "yes" && ac_cv_c_lto_cross_compile="no"
     AC_C_LTO
@@ -836,6 +900,8 @@ AC_MSG_RESULT([${enable_lto}])
 
 # -----------------------------------------------------------------------------
 
+AM_CONDITIONAL([ENABLE_CXX_LINKER], [test "${enable_backend_kinesis}" = "yes" -o "${enable_backend_prometheus_remote_write}" = "yes"])
+
 AC_DEFINE_UNQUOTED([NETDATA_USER], ["${with_user}"], [use this user to drop privileged])
 
 varlibdir="${localstatedir}/lib/netdata"
@@ -898,6 +964,8 @@ AC_SUBST([OPTIONAL_XENSTAT_CFLAGS])
 AC_SUBST([OPTIONAL_XENSTAT_LIBS])
 AC_SUBST([OPTIONAL_KINESIS_CFLAGS])
 AC_SUBST([OPTIONAL_KINESIS_LIBS])
+AC_SUBST([OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS])
+AC_SUBST([OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS])
 
 
 AC_CONFIG_FILES([
@@ -908,6 +976,7 @@ AC_CONFIG_FILES([
     backends/Makefile
     backends/opentsdb/Makefile
     backends/prometheus/Makefile
+    backends/prometheus/remote_write/Makefile
     backends/aws_kinesis/Makefile
     collectors/Makefile
     collectors/apps.plugin/Makefile