Add chart labels to Prometheus. (#15099)

3 files changed, 83 insertions, 21 deletions

diff --git a/exporting/README.md b/exporting/README.md
index c6ce32b656..013f86f32e 100644
--- a/exporting/README.md
+++ b/exporting/README.md
@@ -284,7 +284,7 @@ Configure individual connectors and override any global settings with the follow
     and names are human friendly labels (also unique). Most charts and metrics have the same ID and name, but in several
     cases they are different: disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc.
 
--   `send configured labels = yes | no` controls if labels defined in the `[host labels]` section in `netdata.conf`
+-   `send configured labels = yes | no` controls if host labels defined in the `[host labels]` section in `netdata.conf`
     should be sent to the external database
 
 -   `send automatic labels = yes | no` controls if automatically created labels, like `_os_name` or `_architecture`
diff --git a/exporting/WALKTHROUGH.md b/exporting/WALKTHROUGH.md
index 49cf6587b5..86be758e44 100644
--- a/exporting/WALKTHROUGH.md
+++ b/exporting/WALKTHROUGH.md
@@ -74,10 +74,10 @@ this is your first time using Netdata I suggest you take a look around. The amou
 Next I want to draw your attention to a particular endpoint. Navigate to
 <http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes> In your browser. This is the endpoint which
 publishes all the metrics in a format which Prometheus understands. Let's take a look at one of these metrics.
-`netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000` This
-metric is representing several things which I will go in more details in the section on Prometheus. For now understand
-that this metric: `netdata_system_cpu_percentage_average` has several labels: (`chart`, `family`, `dimension`). This
-corresponds with the first cpu chart you see on the Netdata dashboard.
+`netdata_disk_space_GiB_average{chart="disk_space._run",dimension="avail",family="/run",mount_point="/run",filesystem="tmpfs",mount_root="/"} 0.0298195 1684951093000`
+This metric is representing several things which I will go in more details in the section on Prometheus. For now understand
+that this metric: `netdata_disk_space_GiB_average` has several labels: (`chart`, `family`, `dimension`, `mountt_point`, `filesystem`, `mount_root`).
+This corresponds with disk space you see on the Netdata dashboard.
 
 ![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%204.00.45%20PM.png)
 
@@ -138,12 +138,13 @@ As explained we have two key elements in Prometheus metrics. We have the _metric
 granularity between metrics. Let's use our previous example to further explain.
 
 ```conf
-netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000
+netdata_disk_space_GiB_average{chart="disk_space._run",dimension="avail",family="/run",mount_point="/run",filesystem="tmpfs",mount_root="/"} 0.0298195 1684951093000
 ```
 
-Here our metric is `netdata_system_cpu_percentage_average` and our labels are `chart`, `family`, and `dimension`. The
-last two values constitute the actual metric value for the metric type (gauge, counter, etc…). We can begin graphing
-system metrics with this information, but first we need to hook up Prometheus to poll Netdata stats.
+Here our metric is `netdata_disk_space_GiB_average` and our common labels are `chart`, `family`, and `dimension`. The
+last two values constitute the actual metric value for the metric type (gauge, counter, etc…). We also have specific
+label for this chart named `mount_point`,`filesystem`, and `mount_root`. We can begin graphing system metrics with this information,
+but first we need to hook up Prometheus to poll Netdata stats.
 
 Let's move our attention to Prometheus's configuration. Prometheus gets it config from the file located (in our example)
 at `/opt/prometheus/prometheus.yml`. I won't spend an extensive amount of time going over the configuration values
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index 7a1112abdd..0e0e8abf01 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -326,6 +326,53 @@ void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
     rrdlabels_walkthrough_read(host->rrdlabels, format_prometheus_label_callback, &tmp);
 }
 
+/**
+ * Format host labels for the Prometheus exporter
+ * We are using a structure instead a direct buffer to expand options quickly.
+ *
+ * @param labels_buffer is the buffer used to add labels.
+ */
+
+struct format_prometheus_chart_label_callback {
+    BUFFER *labels_buffer;
+};
+
+static int format_prometheus_chart_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    struct format_prometheus_chart_label_callback *d = (struct format_prometheus_chart_label_callback *)data;
+
+    (void)ls;
+
+    if (name[0] == '_' )
+        return 1;
+
+    char k[PROMETHEUS_ELEMENT_MAX + 1];
+    char v[PROMETHEUS_ELEMENT_MAX + 1];
+
+    prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
+    prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
+
+    if (*k && *v) {
+        buffer_sprintf(d->labels_buffer, ",%s=\"%s\"", k, v);
+    }
+    return 1;
+}
+
+void format_chart_labels_prometheus(struct format_prometheus_chart_label_callback *plabel,
+                                    const char *chart,
+                                    const char *family,
+                                    const char *dim,
+                                    RRDSET *st)
+{
+    if (likely(plabel->labels_buffer))
+        buffer_reset(plabel->labels_buffer);
+    else {
+        plabel->labels_buffer = buffer_create(1024, NULL);
+    }
+    buffer_sprintf(plabel->labels_buffer, "chart=\"%s\",dimension=\"%s\",family=\"%s\"", chart, dim, family);
+
+    rrdlabels_walkthrough_read(st->rrdlabels, format_prometheus_chart_label_callback, plabel);
+}
+
 struct host_variables_callback_options {
     RRDHOST *host;
     BUFFER *wb;
@@ -462,9 +509,17 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p
  * @param p parameters for generating the metric string.
  * @param homogeneous a flag for homogeneous charts.
  * @param prometheus_collector a flag for metrics from prometheus collector.
+ * @param chart_labels the dictionary with chart labels
  */
-static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters *p, int homogeneous, int prometheus_collector)
+static void generate_as_collected_prom_metric(BUFFER *wb,
+                                              struct gen_parameters *p,
+                                              int homogeneous,
+                                              int prometheus_collector,
+                                              DICTIONARY *chart_labels)
 {
+    struct format_prometheus_chart_label_callback local_label;
+    local_label.labels_buffer = wb;
+
     buffer_sprintf(wb, "%s_%s", p->prefix, p->context);
 
     if (!homogeneous)
@@ -475,7 +530,11 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
     if (homogeneous)
         buffer_sprintf(wb, ",dimension=\"%s\"", p->dimension);
 
-    buffer_sprintf(wb, ",family=\"%s\"%s} ", p->family, p->labels);
+    buffer_sprintf(wb, ",family=\"%s\"", p->family);
+
+    rrdlabels_walkthrough_read(chart_labels, format_prometheus_chart_label_callback, &local_label);
+
+    buffer_sprintf(wb, "%s} ", p->labels);
 
     if (prometheus_collector)
         buffer_sprintf(
@@ -564,6 +623,10 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
 
     // for each chart
     RRDSET *st;
+
+    static struct format_prometheus_chart_label_callback plabels = {
+        .labels_buffer = NULL,
+    };
     rrdset_foreach_read(st, host) {
 
         if (likely(can_send_rrdset(instance, st, filter))) {
@@ -655,7 +718,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                             if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
                                 buffer_sprintf(wb, "# TYPE %s_%s%s %s\n", prefix, context, suffix, p.type);
 
-                            generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector);
+                            generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector, st->rrdlabels);
                         }
                         else {
                             // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
@@ -673,7 +736,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                                 buffer_sprintf(
                                     wb, "# TYPE %s_%s_%s%s %s\n", prefix, context, dimension, suffix, p.type);
 
-                            generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector);
+                            generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector, st->rrdlabels);
                         }
                     }
                     else {
@@ -694,6 +757,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                                 (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
                                 PROMETHEUS_ELEMENT_MAX);
 
+                            format_chart_labels_prometheus(&plabels, chart, family, dimension, st);
+
                             if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
                                 buffer_sprintf(
                                     wb,
@@ -713,30 +778,26 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
                                 buffer_sprintf(
                                     wb,
-                                    "%s_%s%s%s{chart=\"%s\",dimension=\"%s\",family=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
+                                    "%s_%s%s%s{%s%s} " NETDATA_DOUBLE_FORMAT
                                     " %llu\n",
                                     prefix,
                                     context,
                                     units,
                                     suffix,
-                                    chart,
-                                    dimension,
-                                    family,
+                                    buffer_tostring(plabels.labels_buffer),
                                     labels,
                                     value,
                                     last_time * MSEC_PER_SEC);
                             else
                                 buffer_sprintf(
                                     wb,
-                                    "%s_%s%s%s{chart=\"%s\",dimension=\"%s\",family=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
+                                    "%s_%s%s%s{%s%s} " NETDATA_DOUBLE_FORMAT
                                     "\n",
                                     prefix,
                                     context,
                                     units,
                                     suffix,
-                                    chart,
-                                    dimension,
-                                    family,
+                                    buffer_tostring(plabels.labels_buffer),
                                     labels,
                                     value);
                         }