Implement the main flow for the Exporting Engine (#7149)

* Add top level tests

* Add a skeleton for preparing buffers

* Initialize graphite instance

* Prepare buffers for all instances

* Add Grafite collected value formatter

* Add support for exporting.conf read and parsing

* - Use new exporting_config instead of netdata_config

* Implement Grafite worker

* Disable exporting engine compilation if libuv is not available

* Add mutex locks

- Configure connectors as connector_<type> in sections of exporting.conf

- Change exporting_select_type to check for connector_ fields

* - Override exporting_config structure if there no exporting.conf so that
  look ups don't fail and we maintain backwards compatibility

* Separate fixtures in unit tests

* Test exporting_discard_responce

* Test response receiving

* Test buffer sending

* Test simple connector worker

- Instance section has the format connector:instance_name
  e.g graphite:my_graphite_instance

- Connectors with : in their name e.g graphite:plaintext are reserved
  So graphite:plaintext is not accepted because it would activate an
  instance with name "plaintext"
  It should be graphite:plaintext:instance_name

* - Enable the add_connector_instance to cleanup the internal structure
  by passing NULL,not NULL arguments

* Implement configurable update interval

- Add additional check to verify instance uniqueness across connectors

* Add host and chart filters

* Add the value calculation over a database series

* Add the calculated over stored data graphite connector

* Add tests for graphite connector

* Add JSON connector

* Add tests for JSON formatting functions

* Add OpenTSDB connector

* Add tests for the OpenTSDB connector

* Add temporaty notes to the documentation

8 files changed, 37 insertions, 29 deletions

diff --git a/backends/backends.c b/backends/backends.c
index 120c6e7033..9ade868822 100644
--- a/backends/backends.c
+++ b/backends/backends.c
@@ -269,9 +269,9 @@ void backend_set_kinesis_variables(int *default_port,
 #if HAVE_KINESIS
     *brc = process_json_response;
     if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 #endif
 }
 
@@ -321,9 +321,9 @@ void backend_set_mongodb_variables(int *default_port,
 #if HAVE_MONGOC
     *brc = process_json_response;
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 #endif
 }
 
@@ -344,9 +344,9 @@ void backend_set_json_variables(int *default_port,
     *brc = process_json_response;
 
     if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_json_plaintext;
+        *brf = backends_format_dimension_collected_json_plaintext;
     else
-        *brf = format_dimension_stored_json_plaintext;
+        *brf = backends_format_dimension_stored_json_plaintext;
 }
 
 /**
@@ -366,9 +366,9 @@ void backend_set_opentsdb_http_variables(int *default_port,
     *brc = process_opentsdb_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_opentsdb_http;
+        *brf = backends_format_dimension_collected_opentsdb_http;
     else
-        *brf = format_dimension_stored_opentsdb_http;
+        *brf = backends_format_dimension_stored_opentsdb_http;
 
 }
 
@@ -389,9 +389,9 @@ void backend_set_opentsdb_telnet_variables(int *default_port,
     *brc = process_opentsdb_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_opentsdb_telnet;
+        *brf = backends_format_dimension_collected_opentsdb_telnet;
     else
-        *brf = format_dimension_stored_opentsdb_telnet;
+        *brf = backends_format_dimension_stored_opentsdb_telnet;
 }
 
 /**
@@ -411,9 +411,9 @@ void backend_set_graphite_variables(int *default_port,
     *brc = process_graphite_response;
 
     if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
-        *brf = format_dimension_collected_graphite_plaintext;
+        *brf = backends_format_dimension_collected_graphite_plaintext;
     else
-        *brf = format_dimension_stored_graphite_plaintext;
+        *brf = backends_format_dimension_stored_graphite_plaintext;
 }
 
 /**
@@ -624,6 +624,9 @@ void *backends_main(void *ptr) {
         case BACKEND_TYPE_UNKNOWN: {
             break;
         }
+        default: {
+            break;
+        }
     }
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
diff --git a/backends/backends.h b/backends/backends.h
index 9da04ccee4..b259141555 100644
--- a/backends/backends.h
+++ b/backends/backends.h
@@ -23,9 +23,13 @@ typedef enum backend_types {
     BACKEND_TYPE_JSON,                      // Stores the data using JSON.
     BACKEND_TYPE_PROMETEUS,                 // The user selected to use Prometheus backend
     BACKEND_TYPE_KINESIS,                   // Send message to AWS Kinesis
-    BACKEND_TYPE_MONGODB                    // Send data to MongoDB collection
+    BACKEND_TYPE_MONGODB,                   // Send data to MongoDB collection
+    BACKEND_TYPE_NUM                        // Number of backend types
 } BACKEND_TYPE;
 
+#ifdef ENABLE_EXPORTING
+#include "exporting/exporting_engine.h"
+#endif
 
 typedef int (**backend_response_checker_t)(BUFFER *);
 typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
@@ -38,6 +42,7 @@ extern BACKEND_OPTIONS global_backend_options;
 extern const char *global_backend_prefix;
 
 extern void *backends_main(void *ptr);
+BACKEND_TYPE backend_select_type(const char *type);
 
 extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options);
 
diff --git a/backends/graphite/graphite.c b/backends/graphite/graphite.c
index 805703893b..f75a93a0fd 100644
--- a/backends/graphite/graphite.c
+++ b/backends/graphite/graphite.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // graphite backend
 
-int format_dimension_collected_graphite_plaintext(
+int backends_format_dimension_collected_graphite_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_graphite_plaintext(
     return 1;
 }
 
-int format_dimension_stored_graphite_plaintext(
+int backends_format_dimension_stored_graphite_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/graphite/graphite.h b/backends/graphite/graphite.h
index b7b0930fa9..498a7fcdf9 100644
--- a/backends/graphite/graphite.h
+++ b/backends/graphite/graphite.h
@@ -6,7 +6,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_graphite_plaintext(
+extern int backends_format_dimension_collected_graphite_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -18,7 +18,7 @@ extern int format_dimension_collected_graphite_plaintext(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_graphite_plaintext(
+extern int backends_format_dimension_stored_graphite_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/json/json.c b/backends/json/json.c
index a53c0f1437..0c7cc738f8 100644
--- a/backends/json/json.c
+++ b/backends/json/json.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // json backend
 
-int format_dimension_collected_json_plaintext(
+int backends_format_dimension_collected_json_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -74,7 +74,7 @@ int format_dimension_collected_json_plaintext(
     return 1;
 }
 
-int format_dimension_stored_json_plaintext(
+int backends_format_dimension_stored_json_plaintext(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/json/json.h b/backends/json/json.h
index 11015652e2..78ac376095 100644
--- a/backends/json/json.h
+++ b/backends/json/json.h
@@ -5,7 +5,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_json_plaintext(
+extern int backends_format_dimension_collected_json_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_json_plaintext(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_json_plaintext(
+extern int backends_format_dimension_stored_json_plaintext(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/opentsdb/opentsdb.c b/backends/opentsdb/opentsdb.c
index 6ee559dbf3..965b4c092f 100644
--- a/backends/opentsdb/opentsdb.c
+++ b/backends/opentsdb/opentsdb.c
@@ -6,7 +6,7 @@
 // ----------------------------------------------------------------------------
 // opentsdb backend
 
-int format_dimension_collected_opentsdb_telnet(
+int backends_format_dimension_collected_opentsdb_telnet(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -42,7 +42,7 @@ int format_dimension_collected_opentsdb_telnet(
     return 1;
 }
 
-int format_dimension_stored_opentsdb_telnet(
+int backends_format_dimension_stored_opentsdb_telnet(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -103,7 +103,7 @@ static inline void opentsdb_build_message(BUFFER *b, char *message, const char *
     );
 }
 
-int format_dimension_collected_opentsdb_http(
+int backends_format_dimension_collected_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -151,7 +151,7 @@ int format_dimension_collected_opentsdb_http(
     return 1;
 }
 
-int format_dimension_stored_opentsdb_http(
+int backends_format_dimension_stored_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
diff --git a/backends/opentsdb/opentsdb.h b/backends/opentsdb/opentsdb.h
index b9372d914b..87d9c5cd73 100644
--- a/backends/opentsdb/opentsdb.h
+++ b/backends/opentsdb/opentsdb.h
@@ -5,7 +5,7 @@
 
 #include "backends/backends.h"
 
-extern int format_dimension_collected_opentsdb_telnet(
+extern int backends_format_dimension_collected_opentsdb_telnet(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -17,7 +17,7 @@ extern int format_dimension_collected_opentsdb_telnet(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-extern int format_dimension_stored_opentsdb_telnet(
+extern int backends_format_dimension_stored_opentsdb_telnet(
           BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -31,7 +31,7 @@ extern int format_dimension_stored_opentsdb_telnet(
 
 extern int process_opentsdb_response(BUFFER *b);
 
-int format_dimension_collected_opentsdb_http(
+int backends_format_dimension_collected_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from
@@ -43,7 +43,7 @@ int format_dimension_collected_opentsdb_http(
         , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
 );
 
-int format_dimension_stored_opentsdb_http(
+int backends_format_dimension_stored_opentsdb_http(
         BUFFER *b                 // the buffer to write data to
         , const char *prefix        // the prefix to use
         , RRDHOST *host             // the host this chart comes from