Percentage of group aggregatable at cloud - fixed for backwards compatibility (#15126)

* percentage of group is now aggregatable at cloud across multiple nodes

* do not break backwards compatibility with percentage-of-instance

* calculate the percentage when percentage-of-instance is requested

* increase capability version

4 files changed, 37 insertions, 15 deletions

diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index d5b8c7570e..5ecf25f932 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -244,12 +244,12 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
     //info("RRD2JSON(): %s: END", r->st->id);
 }
 
-
 void rrdr2json_v2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
     RRDR_OPTIONS options = qt->window.options;
 
-    bool expose_gbc = query_target_aggregatable(qt);
+    bool send_fourth_number = query_target_aggregatable(qt);
+    bool fourth_number_is_vh = send_fourth_number && r->vh && query_has_group_by_aggregation_percentage(qt);
 
     buffer_json_member_add_object(wb, "result");
 
@@ -270,8 +270,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
     buffer_json_member_add_uint64(wb, "value", 0);
     buffer_json_member_add_uint64(wb, "arp", 1);
     buffer_json_member_add_uint64(wb, "pa", 2);
-    if(expose_gbc)
-        buffer_json_member_add_uint64(wb, "count", 3);
+    if(send_fourth_number) {
+        if(fourth_number_is_vh)
+            buffer_json_member_add_uint64(wb, "hidden", 3);
+        else
+            buffer_json_member_add_uint64(wb, "count", 3);
+    }
     buffer_json_object_close(wb);
 
     buffer_json_member_add_array(wb, "data");
@@ -286,6 +290,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
         // for each line in the array
         for (i = start; i != end; i += step) {
             NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
+            NETDATA_DOUBLE *ch = fourth_number_is_vh ? &r->vh[i * r->d ] : NULL;
             RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
             NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
             uint32_t *gbc = &r->gbc [ i * r->d ];
@@ -325,8 +330,12 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
                 buffer_json_add_array_item_uint64(wb, o);
 
                 // add the count
-                if(expose_gbc)
-                    buffer_json_add_array_item_uint64(wb, gbc[d]);
+                if(send_fourth_number) {
+                    if(fourth_number_is_vh)
+                        buffer_json_add_array_item_double(wb, ch[d]);
+                    else
+                        buffer_json_add_array_item_uint64(wb, gbc[d]);
+                }
 
                 buffer_json_array_close(wb); // point
             }
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index def26c754d..ca3a41aae2 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -87,7 +87,7 @@ int rrdset2value_api_v1(
 );
 
 static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) {
-    if(unlikely(options & RRDR_OPTION_RETURN_RAW))
+    if(unlikely((options & RRDR_OPTION_RETURN_RAW) && (rrdr_dim_flags & RRDR_DIMENSION_QUERIED)))
         return true;
 
     if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false;
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index c25f0b7194..b050f34073 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -241,6 +241,7 @@ paths:
             A comma separated list of the groupings required.
             All possible values can be combined together, except `selected`. If `selected` is given in the list, all others are ignored.
             The order they are placed in the list is currently ignored.
+            This parameter is also accepted as `group_by[0]` and `group_by[1]` when multiple grouping passes are required.
           required: false
           schema:
             type: array
@@ -261,6 +262,7 @@ paths:
           in: query
           description: |
             A comma separated list of the label keys to group by their values. The order of the labels in the list is respected.
+            This parameter is also accepted as `group_by_label[0]` and `group_by_label[1]` when multiple grouping passes are required.
           required: false
           schema:
             type: string
@@ -271,6 +273,7 @@ paths:
           description: |
             The aggregation function to apply when grouping metrics together.
             When option `raw` is given, `average` and `avg` behave like `sum` and the caller is expected to calculate the average.
+            This parameter is also accepted as `aggregation[0]` and `aggregation[1]` when multiple grouping passes are required.
           required: false
           schema:
             type: string
@@ -280,6 +283,7 @@ paths:
               - avg
               - average
               - sum
+              - percentage
             default: average
         - $ref: '#/components/parameters/scopeNodes'
         - $ref: '#/components/parameters/scopeContexts'
@@ -2741,8 +2745,13 @@ components:
               type: integer
             count:
               description: |
-                The number of metrics aggregated into this point. This exists only when the option `raw` is given to the query.
+                The number of metrics aggregated into this point.
+                This exists only when the option `raw` is given to the query and the final aggregation point is NOT `percentage`.
               type: integer
+            hidden:
+              description: |
+                The sum of the non-selected dimensions aggregated for this group item point.
+                This exists only when the option `raw` is given to the query and the final aggregation method is `percentage`.
         data:
           type: array
           items:
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index fe02826222..551fee4c3a 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -2992,7 +2992,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
         // initialize partial trimming
         r->partial_data_trimming.max_update_every = update_every_max;
         r->partial_data_trimming.expected_after =
-                (!(qt->window.options & RRDR_OPTION_RETURN_RAW) &&
+                (!query_target_aggregatable(qt) &&
                  qt->window.before >= qt->window.now - update_every_max) ?
                 qt->window.before - update_every_max :
                 qt->window.before;
@@ -3171,6 +3171,9 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
     if(!r->vh)
         return;
 
+    if(query_target_aggregatable(r->internal.qt) && query_has_group_by_aggregation_percentage(r->internal.qt))
+        return;
+
     for(size_t i = 0; i < r->n ;i++) {
         NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
         NETDATA_DOUBLE *ch = &r->vh[ i * r->d ];
@@ -3191,7 +3194,10 @@ static void rrdr2rrdr_group_by_calculate_percentage_of_group(RRDR *r) {
     }
 }
 
-static void rrd2rrdr_convert_to_percentage(RRDR *r) {
+static void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
+    if(!(r->internal.qt->window.options & RRDR_OPTION_PERCENTAGE) || query_target_aggregatable(r->internal.qt))
+        return;
+
     size_t global_min_max_values = 0;
     NETDATA_DOUBLE global_min = NAN, global_max = NAN;
 
@@ -3289,8 +3295,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
 
     if(!r_tmp->group_by.r) {
         // v1 query
-        if(options & RRDR_OPTION_PERCENTAGE)
-            rrd2rrdr_convert_to_percentage(r_tmp);
+        rrd2rrdr_convert_values_to_percentage_of_total(r_tmp);
         return r_tmp;
     }
     // v2 query
@@ -3330,7 +3335,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
         if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE)
             aggregation = qt->request.group_by[g].aggregation;
 
-    if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before)
+    if(!query_target_aggregatable(qt) && r->partial_data_trimming.expected_after < qt->window.before)
         rrdr2rrdr_group_by_partial_trimming(r);
 
     // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max
@@ -3422,8 +3427,7 @@ static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
         qt->window.options &= ~RRDR_OPTION_NONZERO;
     }
 
-    if(options & RRDR_OPTION_PERCENTAGE && !(options & RRDR_OPTION_RETURN_RAW))
-        rrd2rrdr_convert_to_percentage(r);
+    rrd2rrdr_convert_values_to_percentage_of_total(r);
 
     // update query instance counts in query host and query context
     {