Implement the main flow for the Exporting Engine (#7149)

* Add top level tests

* Add a skeleton for preparing buffers

* Initialize graphite instance

* Prepare buffers for all instances

* Add Grafite collected value formatter

* Add support for exporting.conf read and parsing

* - Use new exporting_config instead of netdata_config

* Implement Grafite worker

* Disable exporting engine compilation if libuv is not available

* Add mutex locks

- Configure connectors as connector_<type> in sections of exporting.conf

- Change exporting_select_type to check for connector_ fields

* - Override exporting_config structure if there no exporting.conf so that
  look ups don't fail and we maintain backwards compatibility

* Separate fixtures in unit tests

* Test exporting_discard_responce

* Test response receiving

* Test buffer sending

* Test simple connector worker

- Instance section has the format connector:instance_name
  e.g graphite:my_graphite_instance

- Connectors with : in their name e.g graphite:plaintext are reserved
  So graphite:plaintext is not accepted because it would activate an
  instance with name "plaintext"
  It should be graphite:plaintext:instance_name

* - Enable the add_connector_instance to cleanup the internal structure
  by passing NULL,not NULL arguments

* Implement configurable update interval

- Add additional check to verify instance uniqueness across connectors

* Add host and chart filters

* Add the value calculation over a database series

* Add the calculated over stored data graphite connector

* Add tests for graphite connector

* Add JSON connector

* Add tests for JSON formatting functions

* Add OpenTSDB connector

* Add tests for the OpenTSDB connector

* Add temporaty notes to the documentation

44 files changed, 3832 insertions, 47 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9eaa015838..9ff64b1c38 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@ IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
     set(CXX_FORMAT_SECURITY "-Werror=format-security")
     set(CXX_STACK_PROTECTOR "-fstack-protector-all")
     set(CXX_FLAGS_DEBUG "-O0")
+    set(CMAKE_C_STANDARD 99)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}")
 ELSE()
     message(STATUS "building for: release")
@@ -600,6 +601,23 @@ set(BACKENDS_PLUGIN_FILES
         backends/prometheus/backend_prometheus.h
         )
 
+set(EXPORTING_ENGINE_FILES
+        exporting/exporting_engine.c
+        exporting/exporting_engine.h
+        exporting/graphite/graphite.c
+        exporting/graphite/graphite.h
+        exporting/json/json.c
+        exporting/json/json.h
+        exporting/opentsdb/opentsdb.c
+        exporting/opentsdb/opentsdb.h
+        exporting/read_config.c
+        exporting/init_connectors.c
+        exporting/process_data.c
+        exporting/check_filters.c
+        exporting/send_data.c
+        exporting/send_internal_metrics.c
+        )
+
 set(KINESIS_BACKEND_FILES
         backends/aws_kinesis/aws_kinesis.c
         backends/aws_kinesis/aws_kinesis.h
@@ -639,6 +657,7 @@ set(NETDATA_FILES
         ${DAEMON_FILES}
         ${API_PLUGIN_FILES}
         ${BACKENDS_PLUGIN_FILES}
+        ${EXPORTING_ENGINE_FILES}
         ${CHECKS_PLUGIN_FILES}
         ${HEALTH_PLUGIN_FILES}
         ${IDLEJITTER_PLUGIN_FILES}
@@ -916,6 +935,48 @@ if(BUILD_TESTING)
     target_link_libraries(storage_number_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
     add_test(NAME test_storage_number COMMAND storage_number_testdriver)
 
+    set(EXPORTING_ENGINE_TEST_FILES
+        exporting/tests/test_exporting_engine.c
+        exporting/tests/test_exporting_engine.h
+        exporting/tests/exporting_fixtures.c
+        exporting/tests/exporting_doubles.c
+        exporting/tests/netdata_doubles.c
+        exporting/tests/system_doubles.c
+        )
+    set(TEST_NAME exporting_engine)
+    add_executable(${TEST_NAME}_testdriver ${EXPORTING_ENGINE_TEST_FILES} ${EXPORTING_ENGINE_FILES})
+    target_compile_options(
+        ${TEST_NAME}_testdriver
+        PRIVATE
+        -DUNIT_TESTING
+    )
+    target_link_options(
+        ${TEST_NAME}_testdriver
+        PRIVATE
+        -Wl,--wrap=read_exporting_config
+        -Wl,--wrap=init_connectors
+        -Wl,--wrap=mark_scheduled_instances
+        -Wl,--wrap=rrdhost_is_exportable
+        -Wl,--wrap=rrdset_is_exportable
+        -Wl,--wrap=exporting_calculate_value_from_stored_data
+        -Wl,--wrap=prepare_buffers
+        -Wl,--wrap=notify_workers
+        -Wl,--wrap=send_internal_metrics
+        -Wl,--wrap=now_realtime_sec
+        -Wl,--wrap=uv_thread_create
+        -Wl,--wrap=uv_mutex_lock
+        -Wl,--wrap=uv_mutex_unlock
+        -Wl,--wrap=uv_cond_signal
+        -Wl,--wrap=uv_cond_wait
+        -Wl,--wrap=strdupz
+        -Wl,--wrap=info_int
+        -Wl,--wrap=recv
+        -Wl,--wrap=send
+        -Wl,--wrap=connect_to_one_of
+    )
+    target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
+    add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver)
+
     set(WEB_API_TEST_FILES
         web/api/tests/web_api.c
         web/server/web_client.c
@@ -969,11 +1030,11 @@ if(BUILD_TESTING)
     set_target_properties(
         str2ld_testdriver
         storage_number_testdriver
+        exporting_engine_testdriver
         web_api_testdriver
         valid_urls_testdriver
         PROPERTIES RUNTIME_OUTPUT_DIRECTORY tests
     )
 
-
 endif()
 endif()
diff --git a/Makefile.am b/Makefile.am
index accf2ac691..1158cf073d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -93,6 +93,7 @@ SUBDIRS += \
     collectors \
     daemon \
     database \
+    exporting \
     health \
     libnetdata \
     registry \
@@ -453,6 +454,23 @@ BACKENDS_PLUGIN_FILES = \
     backends/prometheus/backend_prometheus.h \
     $(NULL)
 
+EXPORTING_ENGINE_FILES = \
+    exporting/exporting_engine.c \
+    exporting/exporting_engine.h \
+    exporting/graphite/graphite.c \
+    exporting/graphite/graphite.h \
+    exporting/json/json.c \
+    exporting/json/json.h \
+    exporting/opentsdb/opentsdb.c \
+    exporting/opentsdb/opentsdb.h \
+    exporting/read_config.c \
+    exporting/init_connectors.c \
+    exporting/process_data.c \
+    exporting/check_filters.c \
+    exporting/send_data.c \
+    exporting/send_internal_metrics.c \
+    $(NULL)
+
 KINESIS_BACKEND_FILES = \
     backends/aws_kinesis/aws_kinesis.c \
     backends/aws_kinesis/aws_kinesis.h \
@@ -527,6 +545,12 @@ if LINUX
 
 endif
 
+if ENABLE_EXPORTING
+    NETDATA_FILES += \
+        $(EXPORTING_ENGINE_FILES) \
+        $(NULL)
+endif
+
 NETDATA_COMMON_LIBS = \
     $(OPTIONAL_MATH_LIBS) \
     $(OPTIONAL_ZLIB_LIBS) \
@@ -666,6 +690,7 @@ if ENABLE_UNITTESTS
     check_PROGRAMS = \
         libnetdata/tests/str2ld_testdriver \
         libnetdata/storage_number/tests/storage_number_testdriver \
+        exporting/tests/exporting_engine_testdriver \
         web/api/tests/web_api_testdriver \
         web/api/tests/valid_urls_testdriver \
         $(NULL)
@@ -729,4 +754,46 @@ if ENABLE_UNITTESTS
         $(LIBNETDATA_FILES) \
         $(NULL)
     libnetdata_storage_number_tests_storage_number_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
+
+    EXPORTING_ENGINE_TEST_FILES = \
+        exporting/tests/test_exporting_engine.c \
+        exporting/tests/test_exporting_engine.h \
+        exporting/tests/exporting_fixtures.c \
+        exporting/tests/exporting_doubles.c \
+        exporting/tests/netdata_doubles.c \
+        exporting/tests/system_doubles.c \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_SOURCES = \
+        $(EXPORTING_ENGINE_TEST_FILES) \
+        $(EXPORTING_ENGINE_FILES) \
+        $(LIBNETDATA_FILES) \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_CFLAGS = \
+        $(AM_CFLAGS) \
+        -DUNIT_TESTING \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_LDFLAGS = \
+        -Wl,--wrap=read_exporting_config \
+        -Wl,--wrap=init_connectors \
+        -Wl,--wrap=mark_scheduled_instances \
+        -Wl,--wrap=rrdhost_is_exportable \
+        -Wl,--wrap=rrdset_is_exportable \
+        -Wl,--wrap=exporting_calculate_value_from_stored_data \
+        -Wl,--wrap=prepare_buffers \
+        -Wl,--wrap=notify_workers \
+        -Wl,--wrap=send_internal_metrics \
+        -Wl,--wrap=now_realtime_sec \
+        -Wl,--wrap=uv_thread_create \
+        -Wl,--wrap=uv_mutex_lock \
+        -Wl,--wrap=uv_mutex_unlock \
+        -Wl,--wrap=uv_cond_signal \
+        -Wl,--wrap=uv_cond_wait \
+        -Wl,--wrap=strdupz \
+        -Wl,--wrap=info_int \
+        -Wl,--wrap=recv \
+        -Wl,--wrap=send \
+        -Wl,--wrap=connect_to_one_of \
+        $(TEST_LDFLAGS) \
+        $(NULL)
+    exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
 endif
diff --git a/backends/backends.c b/backends/backends.c
index 120c6e7033..9ade868822 100644
--- a/backends/backends.c
+++ b/backends/backends.c
@@ -269,9 +269,9 @@ void backend_set_kinesis_variables(int *default_port,
 #if HAVE_KINESIS
     *brc = process_json_response;
     if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 #endif
 }
 
@@ -321,9 +321,9 @@ void backend_set_mongodb_variables(int *default_port,
 #if HAVE_MONGOC
     *brc = process_json_response;
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 #endif
 }
 
@@ -344,9 +344,9 @@ void backend_set_json_variables(int *default_port,
     *brc = process_json_response;
 
     if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 }
 
 /**
@@ -366,9 +366,9 @@ void backend_set_opentsdb_http_variables(int *default_port,
     *brc = process_opentsdb_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_opentsdb_http;
+        *brf = backends_format_dimension_collected_opentsdb_http;
     else
-        *brf = format_dimension_stored_opentsdb_http;
+        *brf = backends_format_dimension_stored_opentsdb_http;
 
 }
 
@@ -389,9 +389,9 @@ void backend_set_opentsdb_telnet_variables(int *default_port,
     *brc = process_opentsdb_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_opentsdb_telnet;
+        *brf = backends_format_dimension_collected_opentsdb_telnet;
     else
-        *brf = format_dimension_stored_opentsdb_telnet;
+        *brf = backends_format_dimension_stored_opentsdb_telnet;
 }
 
 /**
@@ -411,9 +411,9 @@ void backend_set_graphite_variables(int *default_port,
     *brc = process_graphite_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_graphite_plaintext;
+        *brf = backends_format_dimension_collected_graphite_plaintext;
     else
-        *brf = format_dimension_stored_graphite_plaintext;
+        *brf = backends_format_dimension_stored_graphite_plaintext;
 }
 
 /**
@@ -624,6 +624,9 @@ void *backends_main(void *ptr) {
         case BACKEND_TYPE_UNKNOWN: {
             break;
         }
+        default: {
+            break;
+        }
     }
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
diff --git a/backends/backends.h b/backends/backends.h
index 9da04ccee4..b259141555 100644
--- a/backends/backends.h
+++ b/backends/backends.h
@@ -23,9 +23,13 @@ typedef enum backend_types {
     BACKEND_TYPE_JSON,                      // Stores the data using JSON.
     BACKEND_TYPE_PROMETEUS,                 // The user selected to use Prometheus backend
     BACKEND_TYPE_KINESIS,                   // Send message to AWS Kinesis
-    BACKEND_TYPE_MONGODB                    // Send data to MongoDB collection
+    BACKEND_TYPE_MONGODB,                   // Send data to MongoDB collection
+    BACKEND_TYPE_NUM                        // Number of backend types
 } BACKEND_TYPE;
 
+#ifdef ENABLE_EXPORTING
+#include "exporting/exporting_engine.h"
+#endif
 
 typedef int (**backend_response_checker_t)(BUFFER *);
 typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
@@ -38,6 +42,7 @@ extern BACKEND_OPTIONS global_backend_options;
 extern const char *global_backend_prefix;
 
 extern void *backends_main(void *ptr);
+BACKEND_TYPE backend_select_type(const char *type);
 
 extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options);
 
diff --git a/backends/graphite/graphite.c b/backends/graphite/graphite.c
index 805703893b..f75a93a0fd 100644
--- a/backends/graphite/graphite.c
+++ b/backends/graphite/graphite.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // graphite backend
 
-int format_dimension_collected_graphite_plaintext(
+int backends_format_dimension_collected_graphite_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_graphite_plaintext(
     return 1;
 }
 
-int format_dimension_stored_graphite_plaintext(
+int backends_format_dimension_stored_graphite_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/graphite/graphite.h b/backends/graphite/graphite.h
index b7b0930fa9..498a7fcdf9 100644
--- a/backends/graphite/graphite.h
+++ b/backends/graphite/graphite.h
@@ -6,7 +6,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_graphite_plaintext(
+extern int backends_format_dimension_collected_graphite_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -18,7 +18,7 @@ extern int format_dimension_collected_graphite_plaintext(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_graphite_plaintext(
+extern int backends_format_dimension_stored_graphite_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/json/json.c b/backends/json/json.c
index a53c0f1437..0c7cc738f8 100644
--- a/backends/json/json.c
+++ b/backends/json/json.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // json backend
 
-int format_dimension_collected_json_plaintext(
+int backends_format_dimension_collected_json_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -74,7 +74,7 @@ int format_dimension_collected_json_plaintext(
     return 1;
 }
 
-int format_dimension_stored_json_plaintext(
+int backends_format_dimension_stored_json_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/json/json.h b/backends/json/json.h
index 11015652e2..78ac376095 100644
--- a/backends/json/json.h
+++ b/backends/json/json.h
@@ -5,7 +5,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_json_plaintext(
+extern int backends_format_dimension_collected_json_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_json_plaintext(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_json_plaintext(
+extern int backends_format_dimension_stored_json_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/opentsdb/opentsdb.c b/backends/opentsdb/opentsdb.c
index 6ee559dbf3..965b4c092f 100644
--- a/backends/opentsdb/opentsdb.c
+++ b/backends/opentsdb/opentsdb.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // opentsdb backend
 
-int format_dimension_collected_opentsdb_telnet(
+int backends_format_dimension_collected_opentsdb_telnet(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_opentsdb_telnet(
     return 1;
 }
 
-int format_dimension_stored_opentsdb_telnet(
+int backends_format_dimension_stored_opentsdb_telnet(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -103,7 +103,7 @@ static inline void opentsdb_build_message(BUFFER *b, char *message, const char *
     );
 }
 
-int format_dimension_collected_opentsdb_http(
+int backends_format_dimension_collected_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -151,7 +151,7 @@ int format_dimension_collected_opentsdb_http(
     return 1;
 }
 
-int format_dimension_stored_opentsdb_http(
+int backends_format_dimension_stored_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/opentsdb/opentsdb.h b/backends/opentsdb/opentsdb.h
index b9372d914b..87d9c5cd73 100644
--- a/backends/opentsdb/opentsdb.h
+++ b/backends/opentsdb/opentsdb.h
@@ -5,7 +5,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_opentsdb_telnet(
+extern int backends_format_dimension_collected_opentsdb_telnet(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_opentsdb_telnet(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_opentsdb_telnet(
+extern int backends_format_dimension_stored_opentsdb_telnet(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -31,7 +31,7 @@ extern int format_dimension_stored_opentsdb_telnet(
 
 extern int process_opentsdb_response(BUFFER *b);
 
-int format_dimension_collected_opentsdb_http(
+int backends_format_dimension_collected_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -43,7 +43,7 @@ int format_dimension_collected_opentsdb_http(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-int format_dimension_stored_opentsdb_http(
+int backends_format_dimension_stored_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/configure.ac b/configure.ac
index 4ae08bffc9..6fcb3baeba 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,6 +36,7 @@ m4_ifdef([AM_SILENT_RULES], [
     ])
 AC_CANONICAL_HOST
 AC_PROG_CC
+AC_PROG_CC_C99
 AM_PROG_CC_C_O
 AC_PROG_CXX
 AC_PROG_INSTALL
@@ -408,6 +409,20 @@ AC_MSG_RESULT([${enable_https}])
 AM_CONDITIONAL([ENABLE_HTTPS], [test "${enable_https}" = "yes"])
 
 # -----------------------------------------------------------------------------
+# Exporting engine
+AC_MSG_CHECKING([if netdata exporting engine should be used])
+if test "${UV_LIBS}"; then
+    enable_exporting_engine="yes"
+    AC_DEFINE([ENABLE_EXPORTING], [1], [netdata exporting engine usability])
+    OPTIONAL_UV_CFLAGS="${UV_CFLAGS}"
+    OPTIONAL_UV_LIBS="${UV_LIBS}"
+else
+    enable_exporting_engine="no"
+fi
+AC_MSG_RESULT([${enable_exporting_engine}])
+AM_CONDITIONAL([ENABLE_EXPORTING], [test "${enable_exporting_engine}" = "yes"])
+
+# -----------------------------------------------------------------------------
 # JSON-C
 test "${enable_jsonc}" = "yes" -a -z "${JSONC_LIBS}" && \
     AC_MSG_ERROR([JSON-C required but not found. Try installing 'libjson-c-dev' or 'json-c'.])
@@ -1209,6 +1224,11 @@ AC_CONFIG_FILES([
     database/Makefile
     database/engine/Makefile
     diagrams/Makefile
+    exporting/Makefile
+    exporting/graphite/Makefile
+    exporting/json/Makefile
+    exporting/opentsdb/Makefile
+    exporting/tests/Makefile
     health/Makefile
     health/notifications/Makefile
     libnetdata/Makefile
diff --git a/daemon/common.h b/daemon/common.h
index df36a49d4e..c04c8ad04f 100644
--- a/daemon/common.h
+++ b/daemon/common.h
@@ -27,7 +27,6 @@
 
 #define config_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed)
 
-
 // ----------------------------------------------------------------------------
 // netdata include files
 
diff --git a/daemon/main.c b/daemon/main.c
index ee2a437fd0..f8c6d84d45 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -75,6 +75,9 @@ struct netdata_static_thread static_threads[] = {
 
         // common plugins for all systems
     {"BACKENDS",             NULL,                    NULL,         1, NULL, NULL, backends_main},
+#ifdef ENABLE_EXPORTING
+    {"EXPORTING",            NULL,                    NULL,         1, NULL, NULL, exporting_main},
+#endif
     {"WEB_SERVER[static1]",  NULL,                    NULL,         0, NULL, NULL, socket_listen_main_static_threaded},
     {"STREAM",               NULL,                    NULL,         0, NULL, NULL, rrdpush_sender_thread},
 
@@ -1105,6 +1108,7 @@ int main(int argc, char **argv) {
     if(!config_loaded)
         load_netdata_conf(NULL, 0);
 
+
     // ----------------------------------------------------------------