Add additional metadata to the data response (#13036)

* Consolidate query params

* Add new option to show full dimensions in the json header (this will include dimensions, charts and chart labels)

* Group and pass parameters with query_params

5 files changed, 132 insertions, 37 deletions

diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 264377e203..6478fee79d 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -2,9 +2,26 @@
 
 #include "json_wrapper.h"
 
+struct value_output {
+    int c;
+    BUFFER *wb;
+};
+
+static int value_list_output(void *entry, void *data) {
+    struct value_output *ap = (struct value_output *)data;
+    BUFFER *wb = ap->wb;
+    char *output = (char *) entry;
+    if(ap->c) buffer_strcat(wb, ",");
+    buffer_strcat(wb, output);
+    (ap->c)++;
+    return 0;
+}
+
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-                             struct context_param *context_param_list, char *chart_label_key)
+    QUERY_PARAMS *rrdset_query_data)
 {
+    struct context_param *context_param_list = rrdset_query_data->context_param_list;
+    char *chart_label_key = rrdset_query_data->chart_label_key;
 
     RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
     int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
@@ -98,6 +115,61 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
     }
     buffer_strcat(wb, "],\n");
 
+    if (rrdset_query_data->show_dimensions) {
+        buffer_sprintf(wb, "   %sfull_dimension_list%s: [", kq, kq);
+
+        char name[RRD_ID_LENGTH_MAX * 2 + 2];
+        char output[RRD_ID_LENGTH_MAX * 2 + 8];
+        char value[RRD_ID_LENGTH_MAX * 2 + 1];
+
+        struct value_output co = {.c = 0, .wb = wb};
+
+        DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
+        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
+            dictionary_set(dict, name, output, len+1);
+        }
+        dictionary_get_all(dict, value_list_output, &co);
+        dictionary_destroy(dict);
+
+        co.c = 0;
+        buffer_sprintf(wb, "],\n   %sfull_chart_list%s: [", kq, kq);
+        dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
+        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
+            dictionary_set(dict, name, output, len + 1);
+        }
+
+        dictionary_get_all(dict, value_list_output, &co);
+        dictionary_destroy(dict);
+
+        RRDSET *st;
+        co.c = 0;
+        buffer_sprintf(wb, "],\n   %sfull_chart_labels%s: [", kq, kq);
+        dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
+        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
+            st = rd->rrdset;
+            if (likely(st->state)) {
+                struct label_index *labels = &st->state->labels;
+                if (labels->head) {
+                    netdata_rwlock_rdlock(&labels->labels_rwlock);
+                    for (struct label *label = labels->head; label; label = label->next) {
+                        sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
+                        int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
+                        snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
+                        dictionary_set(dict, name, output, len + 1);
+                    }
+                    netdata_rwlock_unlock(&labels->labels_rwlock);
+                }
+            }
+        }
+        dictionary_get_all(dict, value_list_output, &co);
+        dictionary_destroy(dict);
+        buffer_strcat(wb, "],\n");
+    }
+
     // Composite charts
     if (context_mode && temp_rd) {
         buffer_sprintf(
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index 14662db740..65dbd5b658 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -5,7 +5,8 @@
 
 #include "rrd2json.h"
 
-extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
+extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
+    QUERY_PARAMS *query_params);
 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 9ac54f7589..1de6be4e33 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -211,7 +211,7 @@ cleanup:
 int rrdset2anything_api_v1(
           ONEWAYALLOC *owa
         , RRDSET *st
-        , BUFFER *wb
+        , QUERY_PARAMS *query_params
         , BUFFER *dimensions
         , uint32_t format
         , long points
@@ -221,16 +221,24 @@ int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
-        , struct context_param *context_param_list
-        , char *chart_label_key
-        , int max_anomaly_rates
-        , int timeout
 )
 {
-    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
+    BUFFER *wb = query_params->wb;
+    if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
         st->last_accessed_time = now_realtime_sec();
 
-    RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
+    RRDR *r = rrd2rrdr(
+        owa,
+        st,
+        points,
+        after,
+        before,
+        group_method,
+        group_time,
+        options,
+        dimensions ? buffer_tostring(dimensions) : NULL,
+        query_params->context_param_list,
+        query_params->timeout);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -242,9 +250,9 @@ int rrdset2anything_api_v1(
     }
 
     if (st && st->state && st->state->is_ar_chart)
-        ml_process_rrdr(r, max_anomaly_rates);
+        ml_process_rrdr(r, query_params->max_anomaly_rates);
 
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+    RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
 
     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
         buffer_no_cacheable(wb);
@@ -258,7 +266,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -271,7 +279,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -284,7 +292,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
             rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
@@ -297,7 +305,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -310,7 +318,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -323,7 +331,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
             buffer_strcat(wb, "[\n");
             rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
@@ -340,7 +348,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -353,7 +361,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
             rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -371,9 +379,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
 
-        rrdr2json(r, wb, options, 1, context_param_list);
+        rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -383,9 +391,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
 
-        rrdr2json(r, wb, options, 1, context_param_list);
+        rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -394,9 +402,9 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
 
-        rrdr2json(r, wb, options, 0, context_param_list);
+        rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -407,9 +415,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
 
-        rrdr2json(r, wb, options, 0, context_param_list);
+        rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index a3fe50cbbb..60bed5b903 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -4,6 +4,17 @@
 #define NETDATA_RRD2JSON_H 1
 
 #include "web/api/web_api_v1.h"
+
+typedef struct query_params {
+    struct context_param *context_param_list;
+    BUFFER *wb;
+    char *chart_label_key;
+    int max_anomaly_rates;
+    int timeout;
+    int show_dimensions;
+} QUERY_PARAMS;
+
+
 #include "web/api/exporters/allmetrics.h"
 #include "web/api/queries/rrdr.h"
 
@@ -56,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
 extern int rrdset2anything_api_v1(
           ONEWAYALLOC *owa
         , RRDSET *st
-        , BUFFER *wb
-        , BUFFER *dimensions
+        ,
+    QUERY_PARAMS *query_params, BUFFER *dimensions
         , uint32_t format
         , long points
         , long long after
@@ -66,10 +77,6 @@ extern int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
-        , struct context_param *context_param_list
-        , char *chart_label_key
-        , int max_anomaly_rates
-        , int timeout
 );
 
 extern int rrdset2value_api_v1(
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index f23d06ac11..be9cac2a03 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -422,6 +422,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     char *chart_labels_filter = NULL;
 
     int group = RRDR_GROUPING_AVERAGE;
+    int show_dimensions = 0;
     uint32_t format = DATASOURCE_JSON;
     uint32_t options = 0x00000000;
 
@@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
         }
+        else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
         else if(!strcmp(name, "after")) after_str = value;
         else if(!strcmp(name, "before")) before_str = value;
         else if(!strcmp(name, "points")) points_str = value;
@@ -645,10 +647,15 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         buffer_strcat(w->response.data, "(");
     }
 
-    ret = rrdset2anything_api_v1(owa, st, w->response.data, dimensions, format,
-                                 points, after, before, group, group_time,
-                                 options, &last_timestamp_in_data, context_param_list,
-                                 chart_label_key, max_anomaly_rates, timeout);
+    QUERY_PARAMS query_params = {
+        .context_param_list = context_param_list,
+        .timeout = timeout,
+        .max_anomaly_rates = max_anomaly_rates,
+        .show_dimensions = show_dimensions,
+        .wb = w->response.data};
+
+    ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
+            points, after, before, group, group_time, options, &last_timestamp_in_data);
 
     free_context_param_list(owa, &context_param_list);