/api/v2/X improvements part 3 (#14665)

* max web request size to 64KB

* fix the request too big message

* increase max request reading tries to 100

* support for bigger web requests

* add "avg" as a shortcut for "average" to both group by aggregation and time aggregation; discard the last partial points of a query in play mode, up to max update every; group by hidden dimensions too

* better implementation for partial data trimming

* added group_by=selected to return only one dimension for all selected metrics

* fix acceptance of group_by=selected

* passing option "raw" disables partial data trimming

* remove obsolete option "plan"; use "debug"

* fix view.min and view.max calculation - there were 2 bugs: a) min and max were reset for every row and b) min and max were corrupted by GBC and AR printing

* per row annotations

* added time column to point annotations

* disable caching for /api/v2/contexts responses

* added api format json2 that returns an array for each points, having all the point values and annotations in them

* work on swagger about /api/v2

* prevent infinite loop

* cleanup and swagger work

* allow negative simple pattern expressions to work as expected

* do not lookup in the dictionary empty names

* garbage collect dictionaries

* make query_target allocate less aggressively; queries fill the remaining points with nulls

* reusable query ops to save memory on huge queries

* move parts of query plans into query ops to save query target memory

* remove storage engine from query metric tiers, to save memory, and recalculate it when it is needed

17 files changed, 1221 insertions, 325 deletions

diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 9da2afec28..0c4580fb99 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -84,7 +84,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
             buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
         }
 
-        int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
             for(c = 0; c < used ;c++) {
@@ -99,7 +98,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
             }
             // prevent a division by zero
             if(total == 0) total = 1;
-            set_min_max = 1;
         }
 
         // for each dimension
@@ -124,13 +122,13 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
                     n = n * 100 / total;
 
-                    if(unlikely(set_min_max)) {
+                    if(unlikely(i == start && c == 0)) {
                         r->view.min = r->view.max = n;
-                        set_min_max = 0;
                     }
-
-                    if(n < r->view.min) r->view.min = n;
-                    if(n > r->view.max) r->view.max = n;
+                    else {
+                        if (n < r->view.min) r->view.min = n;
+                        if (n > r->view.max) r->view.max = n;
+                    }
                 }
 
                 buffer_print_netdata_double(wb, n);
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 016d334e2c..832344ec05 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -211,8 +211,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             buffer_fast_strcat(wb, post_date, post_date_len);
         }
 
-        int set_min_max = 0;
-        if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
+        if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
             total = 0;
             for(c = 0; c < used ;c++) {
                 if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
@@ -232,7 +231,6 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             }
             // prevent a division by zero
             if(total == 0) total = 1;
-            set_min_max = 1;
         }
 
         // for each dimension
@@ -263,16 +261,16 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
                     n = -n;
 
-                if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
+                if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
                     n = n * 100 / total;
 
-                    if(unlikely(set_min_max)) {
+                    if(unlikely(i == start && c == 0)) {
                         r->view.min = r->view.max = n;
-                        set_min_max = 0;
                     }
-
-                    if(n < r->view.min) r->view.min = n;
-                    if(n > r->view.max) r->view.max = n;
+                    else {
+                        if (n < r->view.min) r->view.min = n;
+                        if (n > r->view.max) r->view.max = n;
+                    }
                 }
 
                 buffer_print_netdata_double(wb, n);
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 52eef16019..5e1edc67d0 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -213,7 +213,7 @@ static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, st
         return;
 
     buffer_json_member_add_object(wb, "sts");
-    if(qt->request.group_by_aggregate_function == RRDR_GROUP_BY_FUNCTION_SUM_COUNT) {
+    if(query_target_aggregatable(qt)) {
         buffer_json_member_add_uint64(wb, "cnt", d->group_points);
 
         if(d->sum != 0.0)
@@ -801,7 +801,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR
         buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points);
     buffer_json_array_close(wb);
 
-    if(options & RRDR_OPTION_SHOW_PLAN)
+    if(options & RRDR_OPTION_DEBUG)
         jsonwrap_query_plan(r, wb);
 }
 
@@ -1061,7 +1061,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
 
                             query_target_data_statistics(wb, qt, &qm->query_stats);
 
-                            if(options & RRDR_OPTION_SHOW_PLAN)
+                            if(options & RRDR_OPTION_DEBUG)
                                 jsonwrap_query_metric_plan(wb, qm);
                         }
                     }
@@ -1248,6 +1248,13 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
         buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
         buffer_json_member_add_time_t(wb, "after", r->view.after);
         buffer_json_member_add_time_t(wb, "before", r->view.before);
+
+        buffer_json_member_add_object(wb, "partial_data_trimming");
+        buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
+        buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
+        buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
+        buffer_json_object_close(wb);
+
         buffer_json_member_add_uint64(wb, "points", rows);
         query_target_combined_units_v2(wb, qt, contexts);
         query_target_combined_chart_type(wb, qt, contexts);
@@ -1266,77 +1273,202 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRD
     buffer_json_object_close(wb);
 }
 
-static void annotations_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
-    const size_t dims = r->d, rows = r->rows;
-    size_t next_d_idx = 0;
-    for(size_t d = 0; d < dims ; d++) {
+//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
+//    const size_t dims = r->d, rows = r->rows;
+//    size_t next_d_idx = 0;
+//    for(size_t d = 0; d < dims ; d++) {
+//        if(!rrdr_dimension_should_be_exposed(r->od[d], options))
+//            continue;
+//
+//        size_t d_idx = next_d_idx++;
+//
+//        size_t t = 0;
+//        while(t < rows) {
+//
+//            // find the beginning
+//            time_t started = 0;
+//            for(; t < rows ;t++) {
+//                RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
+//                if(o & flags) {
+//                    started = r->t[t];
+//                    break;
+//                }
+//            }
+//
+//            if(started) {
+//                time_t ended = 0;
+//                for(; t < rows ;t++) {
+//                    RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
+//                    if(!(o & flags)) {
+//                        ended = r->t[t];
+//                        break;
+//                    }
+//                }
+//
+//                if(!ended)
+//                    ended = r->t[rows - 1];
+//
+//                buffer_json_add_array_item_object(wb);
+//                buffer_json_member_add_string(wb, "t", type);
+//                // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
+//                buffer_json_member_add_uint64(wb, "d", d_idx);
+//                if(started == ended) {
+//                    if(options & RRDR_OPTION_MILLISECONDS)
+//                        buffer_json_member_add_time_t2ms(wb, "x", started);
+//                    else
+//                        buffer_json_member_add_time_t(wb, "x", started);
+//                }
+//                else {
+//                    buffer_json_member_add_array(wb, "x");
+//                    if(options & RRDR_OPTION_MILLISECONDS) {
+//                        buffer_json_add_array_item_time_t2ms(wb, started);
+//                        buffer_json_add_array_item_time_t2ms(wb, ended);
+//                    }
+//                    else {
+//                        buffer_json_add_array_item_time_t(wb, started);
+//                        buffer_json_add_array_item_time_t(wb, ended);
+//                    }
+//                    buffer_json_array_close(wb);
+//                }
+//                buffer_json_object_close(wb);
+//            }
+//        }
+//    }
+//}
+//
+//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
+//    buffer_json_member_add_array(wb, "annotations");
+//
+//    annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
+//    annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
+//    annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
+//
+//    buffer_json_array_close(wb); // annotations
+//}
+
+void rrdr2json_v2(RRDR *r __maybe_unused, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
+    bool expose_gbc = query_target_aggregatable(r->internal.qt);
+
+    buffer_json_member_add_object(wb, "result");
+
+    buffer_json_member_add_array(wb, "labels");
+    buffer_json_add_array_item_string(wb, "time");
+    long d, i;
+    const long used = (long)r->d;
+    for(d = 0, i = 0; d < used ; d++) {
         if(!rrdr_dimension_should_be_exposed(r->od[d], options))
             continue;
 
-        size_t d_idx = next_d_idx++;
+        buffer_json_add_array_item_string(wb, string2str(r->dn[d]));
+        i++;
+    }
+    buffer_json_array_close(wb); // labels
+
+    buffer_json_member_add_object(wb, "point");
+    buffer_json_member_add_uint64(wb, "value", 0);
+    buffer_json_member_add_uint64(wb, "ar", 1);
+    buffer_json_member_add_uint64(wb, "pa", 2);
+    if(expose_gbc)
+        buffer_json_member_add_uint64(wb, "count", 3);
+    buffer_json_object_close(wb);
+
+    buffer_json_member_add_array(wb, "data");
+    if(i) {
+        long start = 0, end = rrdr_rows(r), step = 1;
+        if (!(options & RRDR_OPTION_REVERSED)) {
+            start = rrdr_rows(r) - 1;
+            end = -1;
+            step = -1;
+        }
+
+        // for each line in the array
+        for (i = start; i != end; i += step) {
+            NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+            RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
+            NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
+            uint32_t *gbc = &r->gbc [ i * r->d ];
+            time_t now = r->t[i];
+
+            buffer_json_add_array_item_array(wb); // row
+
+            if (options & RRDR_OPTION_MILLISECONDS)
+                buffer_json_add_array_item_time_ms(wb, now); // the time
+            else
+                buffer_json_add_array_item_time_t(wb, now); // the time
+
+            NETDATA_DOUBLE total = 1;
+            if(unlikely((options & RRDR_OPTION_PERCENTAGE) && !(options & (RRDR_OPTION_INTERNAL_GBC|RRDR_OPTION_INTERNAL_AR)))) {
+                total = 0;
+                for(d = 0; d < used ; d++) {
+                    if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) continue;
 
-        size_t t = 0;
-        while(t < rows) {
+                    NETDATA_DOUBLE n = cn[d];
+                    if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
+                        n = -n;
 
-            // find the beginning
-            time_t started = 0;
-            for(; t < rows ;t++) {
-                RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
-                if(o & flags) {
-                    started = r->t[t];
-                    break;
+                    total += n;
                 }
+
+                // prevent a division by zero
+                if(total == 0) total = 1;
             }
 
-            if(started) {
-                time_t ended = 0;
-                for(; t < rows ;t++) {
-                    RRDR_VALUE_FLAGS o = r->o[t * r->d + d];
-                    if(!(o & flags)) {
-                        ended = r->t[t];
-                        break;
-                    }
-                }
+            for (d = 0; d < used; d++) {
+                if (!rrdr_dimension_should_be_exposed(r->od[d], options))
+                    continue;
+
+                RRDR_VALUE_FLAGS o = co[d];
 
-                if(!ended)
-                    ended = r->t[rows - 1];
+                buffer_json_add_array_item_array(wb); // point
 
-                buffer_json_add_array_item_object(wb);
-                buffer_json_member_add_string(wb, "t", type);
-                // buffer_json_member_add_string(wb, "d", string2str(r->dn[d]));
-                buffer_json_member_add_uint64(wb, "d", d_idx);
-                if(started == ended) {
-                    if(options & RRDR_OPTION_MILLISECONDS)
-                        buffer_json_member_add_time_t2ms(wb, "x", started);
+                // add the value
+                NETDATA_DOUBLE n = cn[d];
+
+                if(o & RRDR_VALUE_EMPTY) {
+                    if (unlikely(options & RRDR_OPTION_NULL2ZERO))
+                        buffer_json_add_array_item_double(wb, 0);
                     else
-                        buffer_json_member_add_time_t(wb, "x", started);
+                        buffer_json_add_array_item_double(wb, NAN);
                 }
                 else {
-                    buffer_json_member_add_array(wb, "x");
-                    if(options & RRDR_OPTION_MILLISECONDS) {
-                        buffer_json_add_array_item_time_t2ms(wb, started);
-                        buffer_json_add_array_item_time_t2ms(wb, ended);
+                    if (unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
+                        n = -n;
+
+                    if (unlikely((options & RRDR_OPTION_PERCENTAGE))) {
+                        n = n * 100 / total;
+                    }
+
+                    if(unlikely(i == start && d == 0)) {
+                        r->view.min = r->view.max = n;
                     }
                     else {
-                        buffer_json_add_array_item_time_t(wb, started);
-                        buffer_json_add_array_item_time_t(wb, ended);
+                        if (n < r->view.min) r->view.min = n;
+                        if (n > r->view.max) r->view.max = n;
                     }
-                    buffer_json_array_close(wb);
+
+                    buffer_json_add_array_item_double(wb, n);
                 }
-                buffer_json_object_close(wb);
+
+                // add the anomaly
+                buffer_json_add_array_item_double(wb, ar[d]);
+
+                // add the point annotations
+                buffer_json_add_array_item_uint64(wb, o);
+
+                // add the count
+                if(expose_gbc)
+                    buffer_json_add_array_item_uint64(wb, gbc[d]);
+
+                buffer_json_array_close(wb); // point
             }
+
+            buffer_json_array_close(wb); // row
         }
     }
-}
-
-void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) {
-    buffer_json_member_add_array(wb, "annotations");
 
-    annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap
-    annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow
-    annotations_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial
+    buffer_json_array_close(wb); // data
 
-    buffer_json_array_close(wb); // annotations
+    buffer_json_object_close(wb); // annotations
 }
 
 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options __maybe_unused) {
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index 9a529c422e..797be68fcd 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -10,7 +10,7 @@ typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, R
 typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
-void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
+void rrdr2json_v2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options);
 
 void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format, RRDR_OPTIONS options, RRDR_TIME_GROUPING group_method);
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 453e633e34..240498ee61 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -11,6 +11,10 @@ inline bool query_target_has_percentage_units(struct query_target *qt) {
     return false;
 }
 
+bool query_target_aggregatable(struct query_target *qt) {
+    return (qt->request.options & RRDR_OPTION_RETURN_RAW);
+}
+
 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
     rrdset2json(st, wb, NULL, NULL, 0);
 }
@@ -20,6 +24,9 @@ const char *rrdr_format_to_string(DATASOURCE_FORMAT format)  {
         case DATASOURCE_JSON:
             return DATASOURCE_FORMAT_JSON;
 
+        case DATASOURCE_JSON2:
+            return DATASOURCE_FORMAT_JSON2;
+
         case DATASOURCE_DATATABLE_JSON:
             return DATASOURCE_FORMAT_DATATABLE_JSON;
 
@@ -174,58 +181,74 @@ static RRDR *data_query_group_by(RRDR *r) {
     if(!qt->group_by.used)
         qt->request.group_by &= ~RRDR_GROUP_BY_LABEL;
 
-    if(!(qt->request.group_by & (RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_LABEL)))
+    if(!(qt->request.group_by & (RRDR_GROUP_BY_NODE | RRDR_GROUP_BY_INSTANCE | RRDR_GROUP_BY_DIMENSION | RRDR_GROUP_BY_LABEL | RRDR_GROUP_BY_SELECTED)))
         qt->request.group_by = RRDR_GROUP_BY_DIMENSION;
 
     int added = 0;
     BUFFER *key = buffer_create(0, NULL);
     QUERY_INSTANCE *last_qi = NULL;
     size_t priority = 0;
-    for(size_t c = 0; c < qt->query.used ;c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+    time_t update_every_max = 0;
+    for(size_t d = 0; d < qt->query.used ; d++) {
+        if(unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED)))
             continue;
 
-        QUERY_METRIC *qm = query_metric(qt, c);
+        QUERY_METRIC *qm = query_metric(qt, d);
         QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id);
-        QUERY_NODE *qn = query_node(qt, qm->link.query_host_id);
+        QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
 
         if(qi != last_qi) {
             priority = 0;
             last_qi = qi;
+
+            time_t update_every = rrdinstance_acquired_update_every(qi->ria);
+            if(update_every > update_every_max)
+                update_every_max = update_every;
         }
         else
             priority++;
 
+        // --------------------------------------------------------------------
         // generate the group by key
 
         buffer_flush(key);
-        if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
-            buffer_fast_strcat(key, "|", 1);
-            buffer_strcat(key, query_metric_id(qt, qm));
+        if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
+            buffer_strcat(key, "__hidden_dimensions__");
         }
-        if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
-            buffer_fast_strcat(key, "|", 1);
-            buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
+        else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
+            buffer_strcat(key, "selected");
         }
-        if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
-            DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
-            for(size_t l = 0; l < qt->group_by.used ;l++) {
+        else {
+            if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
                 buffer_fast_strcat(key, "|", 1);
-                rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+                buffer_strcat(key, query_metric_id(qt, qm));
             }
-        }
-        if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
-            buffer_fast_strcat(key, "|", 1);
-            buffer_strcat(key, qn->rrdhost->machine_guid);
-        }
 
-        // append the units
-        if(query_target_has_percentage_units(qt)) {
-            buffer_fast_strcat(key, "|%", 2);
-        }
-        else {
-            buffer_fast_strcat(key, "|", 1);
-            buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
+            if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
+                buffer_fast_strcat(key, "|", 1);
+                buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
+            }
+
+            if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
+                DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
+                for (size_t l = 0; l < qt->group_by.used; l++) {
+                    buffer_fast_strcat(key, "|", 1);
+                    rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+                }
+            }
+
+            if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
+                buffer_fast_strcat(key, "|", 1);
+                buffer_strcat(key, qn->rrdhost->machine_guid);
+            }
+
+            // append the units
+            if (query_target_has_percentage_units(qt)) {
+                buffer_fast_strcat(key, "|%", 2);
+            } else {
+                buffer_fast_strcat(key, "|", 1);
+                buffer_strcat(key, rrdinstance_acquired_units(qi->ria));
+            }
         }
 
         // lookup the key in the dictionary
@@ -237,66 +260,92 @@ static RRDR *data_query_group_by(RRDR *r) {
 
             *set = pos = added++;
 
+            // ----------------------------------------------------------------
             // generate the dimension id
 
             buffer_flush(key);
-            if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
-                buffer_strcat(key, query_metric_id(qt, qm));
+            if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
+                buffer_strcat(key, "__hidden_dimensions__");
             }
-            if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
-                if(buffer_strlen(key) != 0)
-                    buffer_fast_strcat(key, ",", 1);
-
-                if(qt->request.group_by & RRDR_GROUP_BY_NODE)
-                    buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
-                else
-                    buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
+            else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
+                buffer_strcat(key, "selected");
             }
-            if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
-                DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
-                for(size_t l = 0; l < qt->group_by.used ;l++) {
-                    if(buffer_strlen(key) != 0)
+            else {
+                if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
+                    buffer_strcat(key, query_metric_id(qt, qm));
+                }
+
+                if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
+                    if (buffer_strlen(key) != 0)
                         buffer_fast_strcat(key, ",", 1);
-                    rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+
+                    if (qt->request.group_by & RRDR_GROUP_BY_NODE)
+                        buffer_strcat(key, rrdinstance_acquired_id(qi->ria));
+                    else
+                        buffer_strcat(key, string2str(query_instance_id_fqdn(qt, qi)));
                 }
-            }
-            if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
-                if(buffer_strlen(key) != 0)
-                    buffer_fast_strcat(key, ",", 1);
 
-                buffer_strcat(key, qn->rrdhost->machine_guid);
+                if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
+                    DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
+                    for (size_t l = 0; l < qt->group_by.used; l++) {
+                        if (buffer_strlen(key) != 0)
+                            buffer_fast_strcat(key, ",", 1);
+                        rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+                    }
+                }
+
+                if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
+                    if (buffer_strlen(key) != 0)
+                        buffer_fast_strcat(key, ",", 1);
+
+                    buffer_strcat(key, qn->rrdhost->machine_guid);
+                }
             }
+
             entries[pos].id = string_strdupz(buffer_tostring(key));
 
+            // ----------------------------------------------------------------
             // generate the dimension name
 
             buffer_flush(key);
-            if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
-                buffer_strcat(key, query_metric_name(qt, qm));
+            if(unlikely(r->od[d] & RRDR_DIMENSION_HIDDEN)) {
+                buffer_strcat(key, "__hidden_dimensions__");
             }
-            if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
-                if(buffer_strlen(key) != 0)
-                    buffer_fast_strcat(key, ",", 1);
-
-                if(qt->request.group_by & RRDR_GROUP_BY_NODE)
-                    buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
-                else
-                    buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));
+            else if(unlikely(qt->request.group_by & RRDR_GROUP_BY_SELECTED)) {
+                buffer_strcat(key, "selected");
             }
-            if(qt->request.group_by & RRDR_GROUP_BY_LABEL) {
-                DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
-                for(size_t l = 0; l < qt->group_by.used ;l++) {
-                    if(buffer_strlen(key) != 0)
+            else {
+                if (qt->request.group_by & RRDR_GROUP_BY_DIMENSION) {
+                    buffer_strcat(key, query_metric_name(qt, qm));
+                }
+
+                if (qt->request.group_by & RRDR_GROUP_BY_INSTANCE) {
+                    if (buffer_strlen(key) != 0)
                         buffer_fast_strcat(key, ",", 1);
-                    rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+
+                    if (qt->request.group_by & RRDR_GROUP_BY_NODE)
+                        buffer_strcat(key, rrdinstance_acquired_name(qi->ria));
+                    else
+                        buffer_strcat(key, string2str(query_instance_name_fqdn(qt, qi)));
                 }
-            }
-            if(qt->request.group_by & RRDR_GROUP_BY_NODE) {
-                if(buffer_strlen(key) != 0)
-                    buffer_fast_strcat(key, ",", 1);
 
-                buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
+                if (qt->request.group_by & RRDR_GROUP_BY_LABEL) {
+                    DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria);
+                    for (size_t l = 0; l < qt->group_by.used; l++) {
+                        if (buffer_strlen(key) != 0)
+                            buffer_fast_strcat(key, ",", 1);
+                        rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by.label_keys[l], "[unset]");
+                    }
+                }
+
+                if (qt->request.group_by & RRDR_GROUP_BY_NODE) {
+                    if (buffer_strlen(key) != 0)
+                        buffer_fast_strcat(key, ",", 1);
+
+                    buffer_strcat(key, rrdhost_hostname(qn->rrdhost));
+                }
             }
+
             entries[pos].name = string_strdupz(buffer_tostring(key));
 
             // add the rest of the info
@@ -321,8 +370,8 @@ static RRDR *data_query_group_by(RRDR *r) {
         // copy the dimension flags decided by the query target
         // we need this, because if a dimension is explicitly selected
         // the query target adds to it the non-zero flag
-        qm->status |= RRDR_DIMENSION_GROUPED | r->od[c];
-        entries[pos].od |= RRDR_DIMENSION_GROUPED | r->od[c];
+        qm->status |= RRDR_DIMENSION_GROUPED | r->od[d];
+        entries[pos].od |= RRDR_DIMENSION_GROUPED | r->od[d];
     }
 
     // check if we have multiple units
@@ -362,15 +411,22 @@ static RRDR *data_query_group_by(RRDR *r) {
     r2->stats.result_points_generated = r2->d * r2->n;
 
     // initialize r2 (dimension options, names, and ids)
-    for(size_t c2 = 0; c2 < r2->d ; c2++) {
-        r2->od[c2] = entries[c2].od;
-        r2->di[c2] = entries[c2].id;
-        r2->dn[c2] = entries[c2].name;
-        r2->du[c2] = entries[c2].units;
-        r2->dp[c2] = entries[c2].priority;
-        r2->dgbc[c2] = entries[c2].count;
+    for(size_t d2 = 0; d2 < r2->d ; d2++) {
+        r2->od[d2] = entries[d2].od;
+        r2->di[d2] = entries[d2].id;
+        r2->dn[d2] = entries[d2].name;
+        r2->du[d2] = entries[d2].units;
+        r2->dp[d2] = entries[d2].priority;
+        r2->dgbc[d2] = entries[d2].count;
     }
 
+    r2->partial_data_trimming.max_update_every = update_every_max;
+    r2->partial_data_trimming.expected_after =
+            (!(qt->request.options & RRDR_OPTION_RETURN_RAW) && qt->window.before >= qt->window.now - update_every_max) ?
+            qt->window.before - update_every_max :
+            qt->window.before;
+    r2->partial_data_trimming.trimmed_after = qt->window.before;
+