/api/v2/data - multi-host/context/instance/dimension/label queries (#14564)

* fundamentals for having /api/v2/ working

* use an atomic to prevent writing to internal pipe too much

* first attempt of multi-node, multi-context, multi-chart, multi-dimension queries

* v2 jsonwrap

* first attempt for group by

* cleaned up RRDR and fixed group by

* improvements to /api/v2/api

* query instance may be realloced, so pointers to it get invalid; solved memory leaks

* count of quried metrics in summary information

* provide detailed information about selected, excluded, queried and failed metrics for each entity

* select instances by fqdn too

* add timing information to json output

* link charts to rrdcontexts, if a query comes in and it is found unlinked

* calculate min, max, sum, average, volume, count per metric

* api v2 parameters naming

* renders alerts and units

* render machine_guid and node_id in all sections it is relevant

* unified keys

* group by now takes into account units and when there are multiple units involved, it creates a dimension per unit

* request and detailed are hidden behind an option

* summary includes only a flattened list of alerts

* alert counts per host and instance

* count of grouped metrics per dimension

* added contexts to summary

* added chart title

* added dimension priorities and chart type

* support for multiple group by at the same time

* minor fixes

* labels are now a tree

* keys uniformity

* filtering by alerts, both having a specific alert and having a specific alert in a specific status

* added scope of hosts and contexts

* count of instances on contexts and hosts

* make the api return valid responses even when the response contains no data

* calculate average and contribution % for every item in the summary

* fix compilation warnings

* fix compilation warnings - again

33 files changed, 2163 insertions, 540 deletions

diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 1713c84ad9..9da2afec28 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -5,9 +5,8 @@
 
 void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
-    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    const long used = qt->query.used;
+    const long used = (long)r->d;
 
     // print the csv header
     for(c = 0, i = 0; c < used ; c++) {
@@ -22,7 +21,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
         buffer_strcat(wb, separator);
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
-        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
+        buffer_strcat(wb, string2str(r->dn[c]));
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
         i++;
     }
@@ -126,12 +125,12 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
                     n = n * 100 / total;
 
                     if(unlikely(set_min_max)) {
-                        r->min = r->max = n;
+                        r->view.min = r->view.max = n;
                         set_min_max = 0;
                     }
 
-                    if(n < r->min) r->min = n;
-                    if(n > r->max) r->max = n;
+                    if(n < r->view.min) r->view.min = n;
+                    if(n > r->view.max) r->view.max = n;
                 }
 
                 buffer_print_netdata_double(wb, n);
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index d1e71851ce..016d334e2c 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -42,7 +42,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
         strcpy(post_value,         "}");
         strcpy(post_line,          "]}");
         snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n [\n", kq, kq);
-        strcpy(finish,             "\n  ]\n }");
+        strcpy(finish,             "\n        ]\n    }");
 
         snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
         snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
@@ -69,9 +69,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             dates_with_new = 0;
         }
         if( options & RRDR_OPTION_OBJECTSROWS )
-            strcpy(pre_date, "   {");
+            strcpy(pre_date, "            {");
         else
-            strcpy(pre_date, "   [");
+            strcpy(pre_date, "            [");
         strcpy(pre_label,  ",\"");
         strcpy(post_label, "\"");
         strcpy(pre_value,  ",");
@@ -79,10 +79,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             strcpy(post_line, "}");
         else
             strcpy(post_line, "]");
-        snprintfz(data_begin, 100, "],\n  %sdata%s:[\n", kq, kq);
-        strcpy(finish,             "\n  ]\n }");
+        snprintfz(data_begin, 100, "],\n        %sdata%s:[\n", kq, kq);
+        strcpy(finish,             "\n        ]\n    }");
 
-        buffer_sprintf(wb, "{\n  %slabels%s:[", kq, kq);
+        buffer_sprintf(wb, "{\n        %slabels%s:[", kq, kq);
         buffer_sprintf(wb, "%stime%s", sq, sq);
 
         if( options & RRDR_OPTION_OBJECTSROWS )
@@ -104,9 +104,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
     // -------------------------------------------------------------------------
     // print the JSON header
 
-    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    const long used = qt->query.used;
+    const long used = (long)r->d;
 
     // print the header lines
     for(c = 0, i = 0; c < used ; c++) {
@@ -114,7 +113,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             continue;
 
         buffer_fast_strcat(wb, pre_label, pre_label_len);
-        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
+        buffer_strcat(wb, string2str(r->dn[c]));
         buffer_fast_strcat(wb, post_label, post_label_len);
         i++;
     }
@@ -155,6 +154,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
         NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
         RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
         NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
+        uint32_t *gbc = &r->gbc [ i * r->d ];
 
         time_t now = r->t[i];
 
@@ -218,7 +218,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
                 if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
 
                 NETDATA_DOUBLE n;
-                if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
+                if(unlikely(options & RRDR_OPTION_INTERNAL_GBC))
+                    n = gbc[c];
+                else if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
                     n = ar[c];
                 else
                     n = cn[c];
@@ -239,7 +241,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
                 continue;
 
             NETDATA_DOUBLE n;
-            if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
+            if(unlikely(options & RRDR_OPTION_INTERNAL_GBC))
+                n = gbc[c];
+            else if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
                 n = ar[c];
             else
                 n = cn[c];
@@ -247,9 +251,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             buffer_fast_strcat(wb, pre_value, pre_value_len);
 
             if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
-                buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq);
+                buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq);
 
-            if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
+            if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR | RRDR_OPTION_INTERNAL_GBC))) {
                 if(unlikely(options & RRDR_OPTION_NULL2ZERO))
                     buffer_fast_strcat(wb, "0", 1);
                 else
@@ -263,12 +267,12 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
                     n = n * 100 / total;
 
                     if(unlikely(set_min_max)) {
-                        r->min = r->max = n;
+                        r->view.min = r->view.max = n;
                         set_min_max = 0;
                     }
 
-                    if(n < r->min) r->min = n;
-                    if(n > r->max) r->max = n;
+                    if(n < r->view.min) r->view.min = n;
+                    if(n > r->view.max) r->view.max = n;
                 }
 
                 buffer_print_netdata_double(wb, n);
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index ba71ccc86d..936e7f0c19 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -2,54 +2,54 @@
 
 #include "json_wrapper.h"
 
+static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) {
+    buffer_json_member_add_array(wb, "plans");
+    for (size_t p = 0; p < qm->plan.used; p++) {
+        QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
+
+        buffer_json_add_array_item_object(wb);
+        buffer_json_member_add_uint64(wb, "tr", qp->tier);
+        buffer_json_member_add_time_t(wb, "af", qp->after);
+        buffer_json_member_add_time_t(wb, "bf", qp->before);
+        buffer_json_object_close(wb);
+    }
+    buffer_json_array_close(wb);
+
+    buffer_json_member_add_array(wb, "tiers");
+    for (size_t tier = 0; tier < storage_tiers; tier++) {
+        buffer_json_add_array_item_object(wb);
+        buffer_json_member_add_uint64(wb, "tr", tier);
+        buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s);
+        buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s);
+        buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight);
+        buffer_json_object_close(wb);
+    }
+    buffer_json_array_close(wb);
+}
+
 void jsonwrap_query_plan(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
 
     buffer_json_member_add_object(wb, "query_plan");
     for(size_t m = 0; m < qt->query.used; m++) {
-        QUERY_METRIC *qm = &qt->query.array[m];
-
-        buffer_json_member_add_object(wb, string2str(qm->dimension.id));
-        {
-            buffer_json_member_add_array(wb, "plans");
-            for (size_t p = 0; p < qm->plan.used; p++) {
-                QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
-
-                buffer_json_add_array_item_object(wb);
-                buffer_json_member_add_uint64(wb, "tier", qp->tier);
-                buffer_json_member_add_time_t(wb, "after", qp->after);
-                buffer_json_member_add_time_t(wb, "before", qp->before);
-                buffer_json_object_close(wb);
-            }
-            buffer_json_array_close(wb);
-
-            buffer_json_member_add_array(wb, "tiers");
-            for (size_t tier = 0; tier < storage_tiers; tier++) {
-                buffer_json_add_array_item_object(wb);
-                buffer_json_member_add_uint64(wb, "tier", tier);
-                buffer_json_member_add_time_t(wb, "db_first_time", qm->tiers[tier].db_first_time_s);
-                buffer_json_member_add_time_t(wb, "db_last_time", qm->tiers[tier].db_last_time_s);
-                buffer_json_member_add_int64(wb, "weight", qm->tiers[tier].weight);
-                buffer_json_object_close(wb);
-            }
-            buffer_json_array_close(wb);
-        }
+        QUERY_METRIC *qm = query_metric(qt, m);
+        buffer_json_member_add_object(wb, query_metric_id(qt, qm));
+        jsonwrap_query_metric_plan(wb, qm);
         buffer_json_object_close(wb);
     }
     buffer_json_object_close(wb);
 }
 
-static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    QUERY_TARGET *qt = r->internal.qt;
-    const long query_used = qt->query.used;
-    long c, i;
+static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+    const size_t dimensions = r->d;
+    size_t c, i;
 
     buffer_json_member_add_array(wb, key);
-    for(c = 0, i = 0; c < query_used ; c++) {
+    for(c = 0, i = 0; c < dimensions ; c++) {
         if(!rrdr_dimension_should_be_exposed(r->od[c], options))
             continue;
 
-        buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.name));
+        buffer_json_add_array_item_string(wb, string2str(r->dn[c]));
         i++;
     }
     buffer_json_array_close(wb);
@@ -57,17 +57,16 @@ static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r
     return i;
 }
 
-static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    QUERY_TARGET *qt = r->internal.qt;
-    const long query_used = qt->query.used;
-    long c, i;
+static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+    const size_t dimensions = r->d;
+    size_t c, i;
 
     buffer_json_member_add_array(wb, key);
-    for(c = 0, i = 0; c < query_used ; c++) {
+    for(c = 0, i = 0; c < dimensions ; c++) {
         if(!rrdr_dimension_should_be_exposed(r->od[c], options))
             continue;
 
-        buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.id));
+        buffer_json_add_array_item_string(wb, string2str(r->di[c]));
         i++;
     }
     buffer_json_array_close(wb);
@@ -75,7 +74,7 @@ static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r,
     return i;
 }
 
-static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
     QUERY_TARGET *qt = r->internal.qt;
     const long query_used = qt->query.used;
     long c, i;
@@ -85,8 +84,9 @@ static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR
         if(!rrdr_dimension_should_be_exposed(r->od[c], options))
             continue;
 
-        QUERY_METRIC *qm = &qt->query.array[c];
-        buffer_json_add_array_item_string(wb, string2str(qm->chart.id));
+        QUERY_METRIC *qm = query_metric(qt, c);
+        QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
+        buffer_json_add_array_item_string(wb, string2str(qi->id_fqdn));
         i++;
     }
     buffer_json_array_close(wb);
@@ -94,106 +94,401 @@ static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR
     return i;
 }
 
-struct rrdlabels_formatting_v2 {
-    BUFFER *wb;
-    DICTIONARY *dict;
-};
+static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_counts *metrics) {
+    buffer_json_member_add_object(wb, "ds");
+    buffer_json_member_add_uint64(wb, "sl", metrics->selected);
+    buffer_json_member_add_uint64(wb, "ex", metrics->excluded);
+    buffer_json_member_add_uint64(wb, "qr", metrics->queried);
+    buffer_json_member_add_uint64(wb, "fl", metrics->failed);
+    buffer_json_object_close(wb);
+}
 
-static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
-    struct rrdlabels_formatting_v2 *t = data;
-    BUFFER *wb = t->wb;
-    DICTIONARY *dict = t->dict;
+static inline void query_target_instance_counts(BUFFER *wb, struct query_instances_counts *instances) {
+    buffer_json_member_add_object(wb, "is");
+    buffer_json_member_add_uint64(wb, "sl", instances->selected);
+    buffer_json_member_add_uint64(wb, "ex", instances->excluded);
+    buffer_json_object_close(wb);
+}
 
-    char n[RRD_ID_LENGTH_MAX * 2 + 2];
-    snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
+static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_counts *alerts, const char *name, bool array) {
+    if(array)
+        buffer_json_add_array_item_object(wb);
+    else
+        buffer_json_member_add_object(wb, "al");
+
+    if(name)
+        buffer_json_member_add_string(wb, "nm", name);
+    buffer_json_member_add_uint64(wb, "cl", alerts->clear);
+    buffer_json_member_add_uint64(wb, "wr", alerts->warning);
+    buffer_json_member_add_uint64(wb, "cr", alerts->critical);
+    buffer_json_member_add_uint64(wb, "ot", alerts->other);
+    buffer_json_object_close(wb);
+}
 
-    bool existing = 0;
-    bool *set = dictionary_set(dict, n, &existing, sizeof(bool));
-    if(!*set) {
-        *set = true;
-        buffer_json_add_array_item_array(wb);
-        buffer_json_add_array_item_string(wb, name);
-        buffer_json_add_array_item_string(wb, value);
-        buffer_json_array_close(wb);
-    }
+static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, struct query_data_statistics *d) {
+    buffer_json_member_add_object(wb, "sts");
+//    buffer_json_member_add_double(wb, "min", d->min);
+//    buffer_json_member_add_double(wb, "max", d->max);
+//    buffer_json_member_add_double(wb, "sum", d->sum);
+    buffer_json_member_add_double(wb, "avg", (d->count) ? d->sum / (NETDATA_DOUBLE)d->count : 0.0);
+//    buffer_json_member_add_double(wb, "vol", d->volume);
+    buffer_json_member_add_double(wb, "con", (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0);
+    buffer_json_object_close(wb);
+}
 
-    return 1;
+static void query_target_summary_hosts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
+    buffer_json_member_add_array(wb, key);
+    for (long c = 0; c < (long) qt->hosts.used; c++) {
+        QUERY_HOST *qh = query_host(qt, c);
+        RRDHOST *host = qh->host;
+        buffer_json_add_array_item_object(wb);
+        buffer_json_member_add_string(wb, "mg", host->machine_guid);
+        if(qh->node_id[0])
+            buffer_json_member_add_string(wb, "nd", qh->node_id);
+        buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+        query_target_instance_counts(wb, &qh->instances);
+        query_target_metric_counts(wb, &qh->metrics);
+        query_target_alerts_counts(wb, &qh->alerts, NULL, false);
+        query_target_data_statistics(wb, qt, &qh->query_stats);
+        buffer_json_object_close(wb);
+    }
+    buffer_json_array_close(wb);
 }
 
-static inline void jsonwrap_full_dimension_list(BUFFER *wb, RRDR *r) {
-    QUERY_TARGET *qt = r->internal.qt;
+static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
+    buffer_json_member_add_array(wb, key);
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+
+    struct {
+        struct query_data_statistics query_stats;
+        struct query_instances_counts instances;
+        struct query_metrics_counts metrics;
+        struct query_alerts_counts alerts;
+    } x = {
+            .instances = (struct query_instances_counts){ 0 },
+            .metrics = (struct query_metrics_counts){ 0 },
+            .alerts = (struct query_alerts_counts){ 0 },
+    }, *z;
+
+    for (long c = 0; c < (long) qt->contexts.used; c++) {
+        QUERY_CONTEXT *qc = query_context(qt, c);
+
+        z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), &x, sizeof(x));
+
+        z->instances.selected += qc->instances.selected;
+        z->instances.excluded += qc->instances.selected;
 
+        z->metrics.selected += qc->metrics.selected;
+        z->metrics.excluded += qc->metrics.excluded;
+        z->metrics.queried += qc->metrics.queried;
+        z->metrics.failed += qc->metrics.failed;
+
+        z->alerts.clear += qc->alerts.clear;
+        z->alerts.warning += qc->alerts.warning;
+        z->alerts.critical += qc->alerts.critical;
+
+        query_target_merge_data_statistics(&z->query_stats, &qc->query_stats);
+    }
+
+    size_t unique_contexts = dictionary_entries(dict);
+    dfe_start_read(dict, z) {
+        buffer_json_add_array_item_object(wb);
+        buffer_json_member_add_string(wb, "id", z_dfe.name);
+        query_target_instance_counts(wb, &z->instances);
+        query_target_metric_counts(wb, &z->metrics);
+        query_target_alerts_counts(wb, &z->alerts, NULL, false);
+        query_target_data_statistics(wb, qt, &z->query_stats);
+        buffer_json_object_close(wb);
+    }
+    dfe_done(z);
+    buffer_json_array_close(wb);
+    dictionary_destroy(dict);
+
+    return unique_contexts;
+}
+
+static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
     char name[RRD_ID_LENGTH_MAX * 2 + 2];
 
-    buffer_json_member_add_array(wb, "full_dimension_list");
-    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-    for (long c = 0; c < (long)qt->metrics.used ;c++) {
-        RRDMETRIC_ACQUIRED *rma = qt->metrics.array[c];
+    buffer_json_member_add_array(wb, key);
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+    for (long c = 0; c < (long) qt->instances.used; c++) {
+        QUERY_INSTANCE *qi = query_instance(qt, c);
 
         snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
-                  rrdmetric_acquired_id(rma),
-                  rrdmetric_acquired_name(rma));
+                  rrdinstance_acquired_id(qi->ria),
+                  rrdinstance_acquired_name(qi->ria));
 
         bool existing = 0;
         bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
-        if(!*set) {
+        if (!*set) {
             *set = true;
             buffer_json_add_array_item_array(wb);
-            buffer_json_add_array_item_string(wb, rrdmetric_acquired_id(rma));
-            buffer_json_add_array_item_string(wb, rrdmetric_acquired_name(rma));
+            buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria));
+            buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria));
             buffer_json_array_close(wb);
         }
     }
     dictionary_destroy(dict);
     buffer_json_array_close(wb);
+}
 
-    buffer_json_member_add_array(wb, "full_chart_list");
-    dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-    for (long c = 0; c < (long)qt->instances.used ; c++) {
-        RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
+    buffer_json_member_add_array(wb, key);
+    for (long c = 0; c < (long) qt->instances.used; c++) {
+        QUERY_INSTANCE *qi = query_instance(qt, c);
+        QUERY_HOST *qh = query_host(qt, qi->query_host_id);
+
+        buffer_json_add_array_item_object(wb);
+        buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn));
+        buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn));
+        buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria));
+        buffer_json_member_add_string(wb, "mg", qh->host->machine_guid);
+        if(qh->node_id[0])
+            buffer_json_member_add_string(wb, "nd", qh->node_id);
+        query_target_metric_counts(wb, &qi->metrics);
+        query_target_alerts_counts(wb, &qi->alerts, NULL, false);
+        query_target_data_statistics(wb, qt, &qi->query_stats);
+        buffer_json_object_close(wb);
+    }
+    buffer_json_array_close(wb);
+}
+
+static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2) {
+    char name[RRD_ID_LENGTH_MAX * 2 + 2];
+
+    buffer_json_member_add_array(wb, key);
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+    struct {
+        const char *id;
+        const char *name;
+        struct query_data_statistics query_stats;
+        struct query_metrics_counts metrics;
+    } x, *z;
+    size_t q = 0;
+    for (long c = 0; c < (long) qt->dimensions.used; c++) {
+        QUERY_DIMENSION * qd = query_dimension(qt, c);
+        RRDMETRIC_ACQUIRED *rma = qd->rma;
+
+        QUERY_METRIC *qm = NULL;
+        for( ; q < qt->query.used ;q++) {
+            QUERY_METRIC *tqm = query_metric(qt, q);
+            QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id);
+            if(tqd->rma != rma) break;
+            qm = tqm;
+        }
 
         snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
-                  rrdinstance_acquired_id(ria),
-                  rrdinstance_acquired_name(ria));
+                  rrdmetric_acquired_id(rma),
+                  rrdmetric_acquired_name(rma));
 
-        bool existing = 0;
-        bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
-        if(!*set) {
-            *set = true;
-            buffer_json_add_array_item_array(wb);
-            buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(ria));
-            buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(ria));
-            buffer_json_array_close(wb);
+        x.id = rrdmetric_acquired_id(rma);
+        x.name = rrdmetric_acquired_name(rma);
+        x.metrics = (struct query_metrics_counts){ 0 };
+
+        z = dictionary_set(dict, name, &x, sizeof(x));
+
+        if(qm) {
+            z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0;
+            z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0;
+
+            if(qm->status & RRDR_DIMENSION_QUERIED) {
+                z->metrics.queried++;
+                query_target_merge_data_statistics(&z->query_stats, &qm->query_stats);
+            }
         }
+        else
+            z->metrics.excluded++;
     }
+    dfe_start_read(dict, z) {
+                if(v2) {
+                    buffer_json_add_array_item_object(wb);
+                    buffer_json_member_add_string(wb, "id", z->id);
+                    buffer_json_member_add_string(wb, "nm", z->name);
+                    query_target_metric_counts(wb, &z->metrics);
+                    query_target_data_statistics(wb, qt, &z->query_stats);
+                    buffer_json_object_close(wb);
+                }
+                else {
+                    buffer_json_add_array_item_array(wb);
+                    buffer_json_add_array_item_string(wb, z->id);
+                    buffer_json_add_array_item_string(wb, z->name);
+                    buffer_json_array_close(wb);
+                }
+            }
+    dfe_done(z);
     dictionary_destroy(dict);
     buffer_json_array_close(wb);
+}
+
+struct rrdlabels_formatting_v2 {
+    DICTIONARY *keys;
+    QUERY_INSTANCE *qi;
+    bool v2;
+};
+
+struct rrdlabels_keys_dict_entry {
+    const char *name;
+    DICTIONARY *values;
+    struct query_data_statistics query_stats;
+    struct query_metrics_counts metrics;
+};
 
-    buffer_json_member_add_array(wb, "full_chart_labels");
+struct rrdlabels_key_value_dict_entry {
+    const char *key;
+    const char *value;
+    struct query_data_statistics query_stats;
+    struct query_metrics_counts metrics;
+};
+
+static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
+    struct rrdlabels_formatting_v2 *t = data;
+
+    struct rrdlabels_keys_dict_entry k = {
+            .name = name,
+            .values = NULL,
+            .metrics = (struct query_metrics_counts){ 0 },
+    }, *d = dictionary_set(t->keys, name, &k, sizeof(k));
+
+    if(!d->values)