Exporting/send variables (#13221)

16 files changed, 145 insertions, 11 deletions

diff --git a/exporting/aws_kinesis/aws_kinesis.c b/exporting/aws_kinesis/aws_kinesis.c
index fe4181e3ab..1d89cc79a8 100644
--- a/exporting/aws_kinesis/aws_kinesis.c
+++ b/exporting/aws_kinesis/aws_kinesis.c
@@ -45,6 +45,7 @@ int init_aws_kinesis_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_json_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = NULL;
 
diff --git a/exporting/exporting.conf b/exporting/exporting.conf
index 314e1541ef..f2ec56a521 100644
--- a/exporting/exporting.conf
+++ b/exporting/exporting.conf
@@ -45,6 +45,7 @@
     # send names instead of ids = yes
     # send charts matching = *
     # send hosts matching = localhost *
+    # send variables = no
 
 # [kinesis:my_kinesis_instance]
     # enabled = no
diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 9683ca50b8..2141caa41a 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -26,7 +26,8 @@ typedef enum exporting_options {
     EXPORTING_OPTION_SEND_AUTOMATIC_LABELS  = (1 << 4),
     EXPORTING_OPTION_USE_TLS                = (1 << 5),
 
-    EXPORTING_OPTION_SEND_NAMES             = (1 << 16)
+    EXPORTING_OPTION_SEND_NAMES             = (1 << 16),
+    EXPORTING_OPTION_SEND_VARIABLES         = (1 << 17)
 } EXPORTING_OPTIONS;
 
 #define EXPORTING_OPTIONS_SOURCE_BITS                                                                                  \
@@ -45,6 +46,8 @@ extern const char *global_exporting_prefix;
      (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS &&                                             \
       label_source & RRDLABEL_SRC_AUTO))
 
+#define should_send_variables(instance) (instance->config.options & EXPORTING_OPTION_SEND_VARIABLES)
+
 typedef enum exporting_connector_types {
     EXPORTING_CONNECTOR_TYPE_UNKNOWN,                 // Invalid type
     EXPORTING_CONNECTOR_TYPE_GRAPHITE,                // Send plain text to Graphite
@@ -219,6 +222,7 @@ struct instance {
     int (*start_chart_formatting)(struct instance *instance, RRDSET *st);
     int (*metric_formatting)(struct instance *instance, RRDDIM *rd);
     int (*end_chart_formatting)(struct instance *instance, RRDSET *st);
+    int (*variables_formatting)(struct instance *instance, RRDHOST *host);
     int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
     int (*end_batch_formatting)(struct instance *instance);
 
@@ -280,6 +284,7 @@ void start_host_formatting(struct engine *engine, RRDHOST *host);
 void start_chart_formatting(struct engine *engine, RRDSET *st);
 void metric_formatting(struct engine *engine, RRDDIM *rd);
 void end_chart_formatting(struct engine *engine, RRDSET *st);
+void variables_formatting(struct engine *engine, RRDHOST *host);
 void end_host_formatting(struct engine *engine, RRDHOST *host);
 void end_batch_formatting(struct engine *engine);
 int flush_host_labels(struct instance *instance, RRDHOST *host);
diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 33b80c16cd..8ca094b3b7 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -37,6 +37,7 @@ int init_graphite_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_graphite_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = simple_connector_end_batch;
 
diff --git a/exporting/json/json.c b/exporting/json/json.c
index 91f21e61d1..45a8c9d9fe 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -29,6 +29,7 @@ int init_json_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_json_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = simple_connector_end_batch;
 
@@ -87,6 +88,7 @@ int init_json_http_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_json_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = close_batch_json_http;
 
diff --git a/exporting/mongodb/mongodb.c b/exporting/mongodb/mongodb.c
index aab1770d22..850d07fb30 100644
--- a/exporting/mongodb/mongodb.c
+++ b/exporting/mongodb/mongodb.c
@@ -99,6 +99,7 @@ int init_mongodb_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_json_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = format_batch_mongodb;
 
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index 25e7da5a51..282de2e6b9 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -38,6 +38,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_opentsdb_telnet;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = simple_connector_end_batch;
 
@@ -94,6 +95,7 @@ int init_opentsdb_http_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_opentsdb_http;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = close_batch_json_http;
 
diff --git a/exporting/process_data.c b/exporting/process_data.c
index d9154f81b6..d5138b787b 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -259,6 +259,27 @@ void end_chart_formatting(struct engine *engine, RRDSET *st)
 }
 
 /**
+ * Format variables for every connector instance's buffer
+ *
+ * @param engine an engine data structure.
+ * @param host a data collecting host.
+  */
+void variables_formatting(struct engine *engine, RRDHOST *host)
+{
+    for (struct instance *instance = engine->instance_root; instance; instance = instance->next) {
+        if (instance->scheduled && !instance->skip_host && should_send_variables(instance)) {
+            if (instance->variables_formatting && instance->variables_formatting(instance, host) != 0){ 
+                error("EXPORTING: cannot format variables for %s", instance->config.name);
+                disable_instance(instance);
+                continue;
+            }
+            // sum all variables as one metrics
+            instance->stats.buffered_metrics++;
+        }
+    }
+}
+
+/**
  * End host formatting for every connector instance's buffer
  *
  * @param engine an engine data structure.
@@ -334,7 +355,7 @@ void prepare_buffers(struct engine *engine)
             end_chart_formatting(engine, st);
             rrdset_unlock(st);
         }
-
+        variables_formatting(engine, host);
         end_host_formatting(engine, host);
         rrdhost_unlock(host);
     }
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index f6abaab36c..22b5c1d492 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -97,6 +97,7 @@ int init_prometheus_remote_write_instance(struct instance *instance)
     instance->start_chart_formatting = format_chart_prometheus_remote_write;
     instance->metric_formatting = format_dimension_prometheus_remote_write;
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = format_variables_prometheus_remote_write;
     instance->end_host_formatting = NULL;
     instance->end_batch_formatting = format_batch_prometheus_remote_write;
 
@@ -134,14 +135,6 @@ int init_prometheus_remote_write_instance(struct instance *instance)
     return 0;
 }
 
-/**
- * Format host data for Prometheus Remote Write connector
- *
- * @param instance an instance data structure.
- * @param host a data collecting host.
- * @return Always returns 0.
- */
-
 struct format_remote_write_label_callback {
     struct instance *instance;
     void *write_request;
@@ -161,6 +154,13 @@ static int format_remote_write_label_callback(const char *name, const char *valu
     return 1;
 }
 
+/**
+ * Format host data for Prometheus Remote Write connector
+ *
+ * @param instance an instance data structure.
+ * @param host a data collecting host.
+ * @return Always returns 0.
+ */
 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
 {
     struct simple_connector_data *simple_connector_data =
@@ -177,7 +177,7 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
     add_host_info(
         connector_specific_data->write_request,
         "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
-
+    
     if (unlikely(sending_labels_configured(instance))) {
         struct format_remote_write_label_callback tmp = {
             .write_request = connector_specific_data->write_request,
@@ -321,6 +321,49 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
     return 0;
 }
 
+int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) {
+    struct prometheus_remote_write_variables_callback_options *opts = data;
+
+    if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {        
+        RRDHOST *host = opts->host;
+        struct instance *instance = opts->instance;
+        struct simple_connector_data *simple_connector_data =
+            (struct simple_connector_data *)instance->connector_specific_data;
+        struct prometheus_remote_write_specific_data *connector_specific_data =
+            (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;
+
+        char name[PROMETHEUS_LABELS_MAX + 1];
+        char *suffix = "";
+
+        prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX);
+        snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix);
+
+        NETDATA_DOUBLE value = rrdvar2number(rv);
+        add_variable(connector_specific_data->write_request, name,
+            (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS);
+    }
+
+    return 0;
+}
+
+/**
+ * Format a variable for Prometheus Remote Write connector
+ * 
+ * @param rv a variable.
+ * @param instance an instance data structure.
+ * @return Always returns 0.
+ */ 
+int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host)
+{
+    struct prometheus_remote_write_variables_callback_options opt = {
+        .host = host,
+        .instance = instance,
+        .now = now_realtime_usec(),
+    };
+
+    return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt);
+}
+
 /**
  * Format a batch for Prometheus Remote Write connector
  *
diff --git a/exporting/prometheus/remote_write/remote_write.h b/exporting/prometheus/remote_write/remote_write.h
index d738f5126f..4740772d06 100644
--- a/exporting/prometheus/remote_write/remote_write.h
+++ b/exporting/prometheus/remote_write/remote_write.h
@@ -11,12 +11,19 @@ struct prometheus_remote_write_specific_data {
     void *write_request;
 };
 
+struct prometheus_remote_write_variables_callback_options {
+    RRDHOST *host;
+    time_t now;
+    struct instance *instance;
+};
+
 int init_prometheus_remote_write_instance(struct instance *instance);
 extern void clean_prometheus_remote_write(struct instance *instance);
 
 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host);
 int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st);
 int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd);
+int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host);
 int format_batch_prometheus_remote_write(struct instance *instance);
 
 void prometheus_remote_write_prepare_header(struct instance *instance);
diff --git a/exporting/prometheus/remote_write/remote_write_request.cc b/exporting/prometheus/remote_write/remote_write_request.cc
index cfd61271e6..ecfa11fa8a 100644
--- a/exporting/prometheus/remote_write/remote_write_request.cc
+++ b/exporting/prometheus/remote_write/remote_write_request.cc
@@ -138,6 +138,38 @@ void add_metric(
 }
 
 /**
+ * Adds a metric to a write request
+ *
+ * @param write_request_p the write request
+ * @param name the name of the metric
+ * @param instance the name of the host, the metric belongs to
+ * @param value the value of the metric
+ * @param timestamp the timestamp for the metric in milliseconds
+ */
+void add_variable(
+    void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp)
+{
+    WriteRequest *write_request = (WriteRequest *)write_request_p;
+    TimeSeries *timeseries;
+    Sample *sample;
+    Label *label;
+
+    timeseries = write_request->add_timeseries();
+
+    label = timeseries->add_labels();
+    label->set_name("__name__");
+    label->set_value(name);
+
+    label = timeseries->add_labels();
+    label->set_name("instance");
+    label->set_value(instance);
+
+    sample = timeseries->add_samples();
+    sample->set_value(value);
+    sample->set_timestamp(timestamp);
+}
+
+/**
  * Gets the size of a write request
  *
  * @param write_request_p the write request
diff --git a/exporting/prometheus/remote_write/remote_write_request.h b/exporting/prometheus/remote_write/remote_write_request.h
index 5f242b9418..b253701336 100644
--- a/exporting/prometheus/remote_write/remote_write_request.h
+++ b/exporting/prometheus/remote_write/remote_write_request.h
@@ -20,6 +20,9 @@ void add_metric(
     const char *name, const char *chart, const char *family, const char *dimension,
     const char *instance, const double value, const int64_t timestamp);
 
+void add_variable(
+    void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp);
+
 size_t get_write_request_size(void *write_request_p);
 
 int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size);
diff --git a/exporting/pubsub/pubsub.c b/exporting/pubsub/pubsub.c
index 5a5afbdc27..b218338f19 100644
--- a/exporting/pubsub/pubsub.c
+++ b/exporting/pubsub/pubsub.c
@@ -23,6 +23,7 @@ int init_pubsub_instance(struct instance *instance)
         instance->metric_formatting = format_dimension_stored_json_plaintext;
 
     instance->end_chart_formatting = NULL;
+    instance->variables_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = NULL;
 
diff --git a/exporting/read_config.c b/exporting/read_config.c
index b834e867d1..1cba168262 100644
--- a/exporting/read_config.c
+++ b/exporting/read_config.c
@@ -399,6 +399,11 @@ struct engine *read_exporting_config()
         else
             tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
 
+        if (exporter_get_boolean(instance_name, "send variables", CONFIG_BOOLEAN_YES))
+            tmp_instance->config.options |= EXPORTING_OPTION_SEND_VARIABLES;
+        else 
+            tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_VARIABLES;
+
         if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {
             struct prometheus_remote_write_specific_config *connector_specific_config =
                 callocz(1, sizeof(struct prometheus_remote_write_specific_config));
diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c
index da4f40f0d3..75ab7ba43d 100644
--- a/exporting/tests/exporting_doubles.c
+++ b/exporting/tests/exporting_doubles.c
@@ -156,6 +156,14 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st)
     return mock_type(int);
 }
 
+int __mock_variables_formatting(struct instance *instance, RRDHOST *host)
+{
+    function_called();
+    check_expected_ptr(instance);
+    check_expected_ptr(host);
+    return mock_type(int);
+}
+
 int __mock_end_host_formatting(struct instance *instance, RRDHOST *host)
 {
     function_called();
diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h
index 2072dc64f8..ae0b7df9ae 100644
--- a/exporting/tests/test_exporting_engine.h
+++ b/exporting/tests/test_exporting_engine.h
@@ -120,6 +120,7 @@ int __mock_start_host_formatting(struct instance *instance, RRDHOST *host);
 int __mock_start_chart_formatting(struct instance *instance, RRDSET *st);
 int __mock_metric_formatting(struct instance *instance, RRDDIM *rd);
 int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);
+int __mock_variables_formatting(struct instance *instance, RRDHOST *host);
 int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);
 int __mock_end_batch_formatting(struct instance *instance);