Add a Prometheus Remote Write connector to the exporting engine (#8292)

* Copy files from the Prometheus remote write backend

* Update the documentation

* Rename backend -> exporting

* Add the connector to the Netdata build

* Separate files for the remote write connector

* Add an initializer and formatters

* Read a connector specific configuration option

* Add a separate function for header sending

* Use labels instead of tags

* Separate write request for every instance

* Add unit tests

35 files changed, 2407 insertions, 168 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 67a24b9176..55ddfaaf76 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -634,6 +634,8 @@ set(EXPORTING_ENGINE_FILES
         exporting/json/json.h
         exporting/opentsdb/opentsdb.c
         exporting/opentsdb/opentsdb.h
+        exporting/prometheus/prometheus.c
+        exporting/prometheus/prometheus.h
         exporting/read_config.c
         exporting/init_connectors.c
         exporting/process_data.c
@@ -642,6 +644,13 @@ set(EXPORTING_ENGINE_FILES
         exporting/send_internal_metrics.c
         )
 
+set(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES
+        exporting/prometheus/remote_write/remote_write.c
+        exporting/prometheus/remote_write/remote_write.h
+        exporting/prometheus/remote_write/remote_write_request.cc
+        exporting/prometheus/remote_write/remote_write_request.h
+        )
+
 set(KINESIS_EXPORTING_FILES
         exporting/aws_kinesis/aws_kinesis.c
         exporting/aws_kinesis/aws_kinesis.h
@@ -753,9 +762,9 @@ IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
     message(STATUS "prometheus remote write backend: enabled")
 
     find_package(Protobuf REQUIRED)
-    protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS backends/prometheus/remote_write/remote_write.proto)
+    protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS exporting/prometheus/remote_write/remote_write.proto)
 
-    list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
+    list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
     list(APPEND NETDATA_COMMON_LIBRARIES ${PROTOBUF_LIBRARIES} ${SNAPPY_LIBRARIES})
     list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
     list(APPEND NETDATA_COMMON_CFLAGS ${PROTOBUF_CFLAGS_OTHER} ${SNAPPY_CFLAGS_OTHER})
@@ -1021,6 +1030,17 @@ if(BUILD_TESTING)
         )
     set(TEST_NAME exporting_engine)
     set(KINESIS_LINK_OPTIONS)
+    set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
+if(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
+    list(APPEND EXPORTING_ENGINE_FILES ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
+    list(
+        APPEND PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS
+        -Wl,--wrap=init_write_request
+        -Wl,--wrap=add_host_info
+        -Wl,--wrap=add_label
+        -Wl,--wrap=add_metric
+    )
+endif()
 if(ENABLE_BACKEND_KINESIS)
     list(APPEND EXPORTING_ENGINE_FILES ${KINESIS_EXPORTING_FILES})
     list(
@@ -1062,6 +1082,7 @@ endif()
         -Wl,--wrap=send
         -Wl,--wrap=connect_to_one_of
         ${KINESIS_LINK_OPTIONS}
+        ${PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS}
     )
     target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
     add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver)
diff --git a/Makefile.am b/Makefile.am
index 80fb031fbc..a9b1c6282e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -492,6 +492,8 @@ EXPORTING_ENGINE_FILES = \
     exporting/json/json.h \
     exporting/opentsdb/opentsdb.c \
     exporting/opentsdb/opentsdb.h \
+    exporting/prometheus/prometheus.c \
+    exporting/prometheus/prometheus.h \
     exporting/read_config.c \
     exporting/init_connectors.c \
     exporting/process_data.c \
@@ -500,6 +502,14 @@ EXPORTING_ENGINE_FILES = \
     exporting/send_internal_metrics.c \
     $(NULL)
 
+PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES = \
+    exporting/prometheus/remote_write/remote_write.c \
+    exporting/prometheus/remote_write/remote_write.h \
+    exporting/prometheus/remote_write/remote_write_request.cc \
+    exporting/prometheus/remote_write/remote_write_request.h \
+    exporting/prometheus/remote_write/remote_write.proto \
+    $(NULL)
+
 KINESIS_EXPORTING_FILES = \
     exporting/aws_kinesis/aws_kinesis.c \
     exporting/aws_kinesis/aws_kinesis.h \
@@ -517,7 +527,6 @@ KINESIS_BACKEND_FILES = \
 PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \
     backends/prometheus/remote_write/remote_write.cc \
     backends/prometheus/remote_write/remote_write.h \
-    backends/prometheus/remote_write/remote_write.proto \
     $(NULL)
 
 MONGODB_BACKEND_FILES = \
@@ -737,16 +746,19 @@ if ENABLE_BACKEND_KINESIS
 endif
 
 if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
+if ENABLE_EXPORTING
+    netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
+endif
     netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES)
     netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
     BUILT_SOURCES = \
-        backends/prometheus/remote_write/remote_write.pb.cc \
-        backends/prometheus/remote_write/remote_write.pb.h \
+        exporting/prometheus/remote_write/remote_write.pb.cc \
+        exporting/prometheus/remote_write/remote_write.pb.h \
         $(NULL)
     nodist_netdata_SOURCES = $(BUILT_SOURCES)
 
-backends/prometheus/remote_write/remote_write.pb.cc \
-backends/prometheus/remote_write/remote_write.pb.h: backends/prometheus/remote_write/remote_write.proto
+exporting/prometheus/remote_write/remote_write.pb.cc \
+exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote_write/remote_write.proto
 	$(PROTOC) --proto_path=$(srcdir) --cpp_out=$(builddir) $^
 
 endif
@@ -867,6 +879,17 @@ if ENABLE_UNITTESTS
         $(TEST_LDFLAGS) \
         $(NULL)
     exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
+if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
+    exporting_tests_exporting_engine_testdriver_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
+    exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
+    exporting_tests_exporting_engine_testdriver_LDFLAGS += \
+        -Wl,--wrap=init_write_request \
+        -Wl,--wrap=add_host_info \
+        -Wl,--wrap=add_label \
+        -Wl,--wrap=add_metric \
+        $(NULL)
+    nodist_exporting_tests_exporting_engine_testdriver_SOURCES = $(BUILT_SOURCES)
+endif
 if ENABLE_BACKEND_KINESIS
     exporting_tests_exporting_engine_testdriver_SOURCES += $(KINESIS_EXPORTING_FILES)
     exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_KINESIS_LIBS)
diff --git a/backends/backends.c b/backends/backends.c
index b373b2d05e..e1964bd4a2 100644
--- a/backends/backends.c
+++ b/backends/backends.c
@@ -295,7 +295,7 @@ void backend_set_prometheus_variables(int *default_port,
 #endif
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
-    *brc = process_prometheus_remote_write_response;
+    *brc = backends_process_prometheus_remote_write_response;
 #endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
 }
 
@@ -439,7 +439,7 @@ BACKEND_TYPE backend_select_type(const char *type) {
         return BACKEND_TYPE_JSON;
     }
     else if (!strcmp(type, "prometheus_remote_write")) {
-        return  BACKEND_TYPE_PROMETHEUS;
+        return  BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;
     }
     else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
         return BACKEND_TYPE_KINESIS;
@@ -557,12 +557,12 @@ void *backends_main(void *ptr) {
             backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
             break;
         }
-        case BACKEND_TYPE_PROMETHEUS: {
+        case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE: {
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
             do_prometheus_remote_write = 1;
 
             http_request_header = buffer_create(1);
-            init_write_request();
+            backends_init_write_request();
 #else
             error("BACKEND: Prometheus remote write support isn't compiled");
 #endif // ENABLE_PROMETHEUS_REMOTE_WRITE
@@ -724,7 +724,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
         if(do_prometheus_remote_write)
-            clear_write_request();
+            backends_clear_write_request();
 #endif
         rrd_rdlock();
         RRDHOST *host;
@@ -755,7 +755,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
             if(do_prometheus_remote_write) {
-                rrd_stats_remote_write_allmetrics_prometheus(
+                backends_rrd_stats_remote_write_allmetrics_prometheus(
                     host
                     , __hostname
                     , global_backend_prefix
@@ -1048,7 +1048,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
                 if(do_prometheus_remote_write) {
-                    size_t data_size = get_write_request_size();
+                    size_t data_size = backends_get_write_request_size();
 
                     if(unlikely(!data_size)) {
                         error("BACKEND: write request size is out of range");
@@ -1057,7 +1057,7 @@ void *backends_main(void *ptr) {
 
                     buffer_flush(b);
                     buffer_need_bytes(b, data_size);
-                    if(unlikely(pack_write_request(b->buffer, &data_size))) {
+                    if(unlikely(backends_pack_write_request(b->buffer, &data_size))) {
                         error("BACKEND: cannot pack write request");
                         continue;
                     }
@@ -1137,7 +1137,7 @@ void *backends_main(void *ptr) {
         if(do_prometheus_remote_write && failures) {
             (void) buffer_on_failures;
             failures = 0;
-            chart_lost_bytes = chart_buffered_bytes = get_write_request_size(); // estimated write request size
+            chart_lost_bytes = chart_buffered_bytes = backends_get_write_request_size(); // estimated write request size
             chart_data_lost_events++;
             chart_lost_metrics = chart_buffered_metrics;
         } else
@@ -1209,7 +1209,7 @@ cleanup:
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
     buffer_free(http_request_header);
     if(do_prometheus_remote_write)
-        protocol_buffers_shutdown();
+        backends_protocol_buffers_shutdown();
 #endif
 
 #if HAVE_MONGOC
diff --git a/backends/backends.h b/backends/backends.h
index c0cd235987..212823a078 100644
--- a/backends/backends.h
+++ b/backends/backends.h
@@ -21,7 +21,7 @@ typedef enum backend_types {
     BACKEND_TYPE_OPENTSDB_USING_TELNET,     // Send data to OpenTSDB using telnet API
     BACKEND_TYPE_OPENTSDB_USING_HTTP,       // Send data to OpenTSDB using HTTP API
     BACKEND_TYPE_JSON,                      // Stores the data using JSON.
-    BACKEND_TYPE_PROMETHEUS,                // The user selected to use Prometheus backend
+    BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE,   // The user selected to use Prometheus backend
     BACKEND_TYPE_KINESIS,                   // Send message to AWS Kinesis
     BACKEND_TYPE_MONGODB,                   // Send data to MongoDB collection
     BACKEND_TYPE_NUM                        // Number of backend types
diff --git a/backends/prometheus/backend_prometheus.c b/backends/prometheus/backend_prometheus.c
index 2db911621d..b3f955e15f 100644
--- a/backends/prometheus/backend_prometheus.c
+++ b/backends/prometheus/backend_prometheus.c
@@ -159,7 +159,7 @@ struct host_variables_callback_options {
     RRDHOST *host;
     BUFFER *wb;
     BACKEND_OPTIONS backend_options;
-    PROMETHEUS_OUTPUT_OPTIONS output_options;
+    BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options;
     const char *prefix;
     const char *labels;
     time_t now;
@@ -174,14 +174,14 @@ static int print_host_variables(RRDVAR *rv, void *data) {
         if(!opts->host_header_printed) {
             opts->host_header_printed = 1;
 
-            if(opts->output_options & PROMETHEUS_OUTPUT_HELP) {
+            if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP) {
                 buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n");
             }
         }
 
         calculated_number value = rrdvar2number(rv);
         if(isnan(value) || isinf(value)) {
-            if(opts->output_options & PROMETHEUS_OUTPUT_HELP)
+            if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP)
                 buffer_sprintf(opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value))?"NAN":"INF");
 
             return 0;
@@ -196,7 +196,7 @@ static int print_host_variables(RRDVAR *rv, void *data) {
 
         prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));
 
-        if(opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+        if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
             buffer_sprintf(opts->wb
                            , "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n"
                            , opts->prefix
@@ -223,7 +223,7 @@ static int print_host_variables(RRDVAR *rv, void *data) {
     return 0;
 }
 
-static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, PROMETHEUS_OUTPUT_OPTIONS output_options) {
+static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {
     rrdhost_rdlock(host);
 
     char hostname[PROMETHEUS_ELEMENT_MAX + 1];
@@ -231,13 +231,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
 
     char labels[PROMETHEUS_LABELS_MAX + 1] = "";
     if(allhosts) {
-        if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+        if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
             buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
         else
             buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);
 
         if(host->tags && *(host->tags)) {
-            if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
+            if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
@@ -255,13 +255,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
         snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
     }
     else {
-        if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+        if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
             buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
         else
             buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);
 
         if(host->tags && *(host->tags)) {
-            if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
+            if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
@@ -277,7 +277,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
     }
 
     // send custom variables set for the host
-    if(output_options & PROMETHEUS_OUTPUT_VARIABLES){
+    if(output_options & BACKENDS_PROMETHEUS_OUTPUT_VARIABLES){
         struct host_variables_callback_options opts = {
                 .host = host,
                 .wb = wb,
@@ -299,7 +299,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
         char family[PROMETHEUS_ELEMENT_MAX + 1];
         char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
 
-        prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
+        prometheus_label_copy(chart, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
         prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
         prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
 
@@ -316,14 +316,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                     homogeneous = 0;
             }
             else {
-                if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS))
-                    prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
+                if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS))
+                    prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS);
             }
 
-            if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
+            if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
                 buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n"
                                , (homogeneous)?"homogeneous":"heterogeneous"
-                               , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
+                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
                                , st->context
                                , st->family
                                , st->units
@@ -354,18 +354,18 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                             // all the dimensions of the chart, has the same algorithm, multiplier and divisor
                             // we add all dimensions as labels
 
-                            prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+                            prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
 
-                            if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
+                            if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
                                 buffer_sprintf(wb
                                                , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
                                                , prefix
                                                , context
                                                , suffix
-                                               , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
+                                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
                                                , st->context
                                                , st->family
-                                               , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
+                                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
                                                , rd->multiplier
                                                , rd->divisor
                                                , h
@@ -373,7 +373,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                                                , t
                                 );
 
-                            if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
+                            if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
                                 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n"
                                                , prefix
                                                , context
@@ -381,7 +381,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                                                , t
                                 );
 
-                            if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+                            if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
                                 buffer_sprintf(wb
                                                , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
                                                , prefix
@@ -411,19 +411,19 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                             // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
                             // we create a metric per dimension
 
-                            prometheus_name_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+                            prometheus_name_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
 
-                            if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
+                            if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
                                 buffer_sprintf(wb
                                                , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
                                                , prefix
                                                , context
                                                , dimension
                                                , suffix
-                                               , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
+                                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id
                                                , st->context
                                                , st->family
-                                               , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
+                                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
                                                , rd->multiplier
                                                , rd->divisor
                                                , h
@@ -431,7 +431,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                                                , t
                                 );
 
-                            if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
+                            if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
                                 buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n"
                                                , prefix
                                                , context
@@ -440,7 +440,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                                                , t
                                 );
 
-                            if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+                            if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
                                 buffer_sprintf(wb
                                                , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
                                                , prefix
@@ -480,21 +480,21 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                             else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)
                                 suffix = "_sum";
 
-                            prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+                            prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
 
-                            if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
+                            if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))
                                 buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"
                                                , prefix
                                                , context
                                                , units
                                                , suffix
-                                               , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
+                                               , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id
                                                , st->units
                                                , (unsigned long long)first_t
                                                , (unsigned long long)last_t
                                 );
 
-                            if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
+                            if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))
                                 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n"
                                                , prefix
                                                , context
@@ -502,7 +502,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
                                                , suffix
                                 );
 
-                            if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+                            if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)
                                 buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n"
                                                , prefix
                                                , context
@@ -581,7 +581,7 @@ inline static void remote_write_split_words(char *str, char **words, int max_wor
     }
 }
 
-void rrd_stats_remote_write_allmetrics_prometheus(
+void backends_rrd_stats_remote_write_allmetrics_prometheus(
         RRDHOST *host
         , const char *__hostname
         , const char *prefix
@@ -595,7 +595,7 @@ void rrd_stats_remote_write_allmetrics_prometheus(
     char hostname[PROMETHEUS_ELEMENT_MAX + 1];
     prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX);
 
-    add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
+    backends_add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
 
     if(host->tags && *(host->tags)) {
         char tags[PROMETHEUS_LABELS_MAX + 1];
@@ -605,10 +605,10 @@ void rrd_stats_remote_write_allmetrics_prometheus(
 
         remote_write_split_words(tags, words, PROMETHEUS_LABELS_MAX_NUMBER);
 
-        add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS);
+        backends_add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS);
 
         for(i = 0; words[i] != NULL && words[i + 1] != NULL && (i + 1) < PROMETHEUS_LABELS_MAX_NUMBER; i += 2) {
-            add_tag(words[i], words[i + 1]);