From 6f270819121afb743d783f9a72786e367d1048a3 Mon Sep 17 00:00:00 2001
From: Vladimir Kobal <vlad@prokk.net>
Date: Thu, 12 Dec 2019 21:41:11 +0200
Subject: Implement the main flow for the Exporting Engine (#7149)

* Add top level tests

* Add a skeleton for preparing buffers

* Initialize graphite instance

* Prepare buffers for all instances

* Add Grafite collected value formatter

* Add support for exporting.conf read and parsing

* - Use new exporting_config instead of netdata_config

* Implement Grafite worker

* Disable exporting engine compilation if libuv is not available

* Add mutex locks

- Configure connectors as connector_<type> in sections of exporting.conf

- Change exporting_select_type to check for connector_ fields

* - Override exporting_config structure if there no exporting.conf so that
  look ups don't fail and we maintain backwards compatibility

* Separate fixtures in unit tests

* Test exporting_discard_responce

* Test response receiving

* Test buffer sending

* Test simple connector worker

- Instance section has the format connector:instance_name
  e.g graphite:my_graphite_instance

- Connectors with : in their name e.g graphite:plaintext are reserved
  So graphite:plaintext is not accepted because it would activate an
  instance with name "plaintext"
  It should be graphite:plaintext:instance_name

* - Enable the add_connector_instance to cleanup the internal structure
  by passing NULL,not NULL arguments

* Implement configurable update interval

- Add additional check to verify instance uniqueness across connectors

* Add host and chart filters

* Add the value calculation over a database series

* Add the calculated over stored data graphite connector

* Add tests for graphite connector

* Add JSON connector

* Add tests for JSON formatting functions

* Add OpenTSDB connector

* Add tests for the OpenTSDB connector

* Add temporaty notes to the documentation
---
 Makefile.am | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

(limited to 'Makefile.am')

diff --git a/Makefile.am b/Makefile.am
index accf2ac691..1158cf073d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -93,6 +93,7 @@ SUBDIRS += \
     collectors \
     daemon \
     database \
+    exporting \
     health \
     libnetdata \
     registry \
@@ -453,6 +454,23 @@ BACKENDS_PLUGIN_FILES = \
     backends/prometheus/backend_prometheus.h \
     $(NULL)
 
+EXPORTING_ENGINE_FILES = \
+    exporting/exporting_engine.c \
+    exporting/exporting_engine.h \
+    exporting/graphite/graphite.c \
+    exporting/graphite/graphite.h \
+    exporting/json/json.c \
+    exporting/json/json.h \
+    exporting/opentsdb/opentsdb.c \
+    exporting/opentsdb/opentsdb.h \
+    exporting/read_config.c \
+    exporting/init_connectors.c \
+    exporting/process_data.c \
+    exporting/check_filters.c \
+    exporting/send_data.c \
+    exporting/send_internal_metrics.c \
+    $(NULL)
+
 KINESIS_BACKEND_FILES = \
     backends/aws_kinesis/aws_kinesis.c \
     backends/aws_kinesis/aws_kinesis.h \
@@ -527,6 +545,12 @@ if LINUX
 
 endif
 
+if ENABLE_EXPORTING
+    NETDATA_FILES += \
+        $(EXPORTING_ENGINE_FILES) \
+        $(NULL)
+endif
+
 NETDATA_COMMON_LIBS = \
     $(OPTIONAL_MATH_LIBS) \
     $(OPTIONAL_ZLIB_LIBS) \
@@ -666,6 +690,7 @@ if ENABLE_UNITTESTS
     check_PROGRAMS = \
         libnetdata/tests/str2ld_testdriver \
         libnetdata/storage_number/tests/storage_number_testdriver \
+        exporting/tests/exporting_engine_testdriver \
         web/api/tests/web_api_testdriver \
         web/api/tests/valid_urls_testdriver \
         $(NULL)
@@ -729,4 +754,46 @@ if ENABLE_UNITTESTS
         $(LIBNETDATA_FILES) \
         $(NULL)
     libnetdata_storage_number_tests_storage_number_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
+
+    EXPORTING_ENGINE_TEST_FILES = \
+        exporting/tests/test_exporting_engine.c \
+        exporting/tests/test_exporting_engine.h \
+        exporting/tests/exporting_fixtures.c \
+        exporting/tests/exporting_doubles.c \
+        exporting/tests/netdata_doubles.c \
+        exporting/tests/system_doubles.c \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_SOURCES = \
+        $(EXPORTING_ENGINE_TEST_FILES) \
+        $(EXPORTING_ENGINE_FILES) \
+        $(LIBNETDATA_FILES) \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_CFLAGS = \
+        $(AM_CFLAGS) \
+        -DUNIT_TESTING \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_LDFLAGS = \
+        -Wl,--wrap=read_exporting_config \
+        -Wl,--wrap=init_connectors \
+        -Wl,--wrap=mark_scheduled_instances \
+        -Wl,--wrap=rrdhost_is_exportable \
+        -Wl,--wrap=rrdset_is_exportable \
+        -Wl,--wrap=exporting_calculate_value_from_stored_data \
+        -Wl,--wrap=prepare_buffers \
+        -Wl,--wrap=notify_workers \
+        -Wl,--wrap=send_internal_metrics \
+        -Wl,--wrap=now_realtime_sec \
+        -Wl,--wrap=uv_thread_create \
+        -Wl,--wrap=uv_mutex_lock \
+        -Wl,--wrap=uv_mutex_unlock \
+        -Wl,--wrap=uv_cond_signal \
+        -Wl,--wrap=uv_cond_wait \
+        -Wl,--wrap=strdupz \
+        -Wl,--wrap=info_int \
+        -Wl,--wrap=recv \
+        -Wl,--wrap=send \
+        -Wl,--wrap=connect_to_one_of \
+        $(TEST_LDFLAGS) \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
 endif
-- 
cgit v1.2.3