/api/v2 part 8 (#14885)

* configure extent cache size

* /api/v2/data responsed with the id of dimensions in result.labels

* provide the original units and sts to db.dimensions

* updated swagger for the changes

4 files changed, 130 insertions, 191 deletions

diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 8d7f49da9b..d5b8c7570e 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -261,7 +261,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
         if(!rrdr_dimension_should_be_exposed(r->od[d], options))
             continue;
 
-        buffer_json_add_array_item_string(wb, string2str(r->dn[d]));
+        buffer_json_add_array_item_string(wb, string2str(r->di[d]));
         i++;
     }
     buffer_json_array_close(wb); // labels
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index b19ce2590a..d94ef047d8 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -950,8 +950,8 @@ static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) {
     }
 }
 
-static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
-    if(query_target_has_percentage_units(qt)) {
+static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts, bool ignore_percentage) {
+    if(!ignore_percentage && query_target_has_percentage_units(qt)) {
         buffer_json_member_add_string(wb, "units", "%");
     }
     else if(contexts == 1) {
@@ -1036,11 +1036,11 @@ static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_
     buffer_json_array_close(wb); // group_by_order
 }
 
-static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool ignore_percentage) {
     if(!r->du)
         return;
 
-    bool percentage = query_target_has_percentage_units(r->internal.qt);
+    bool percentage = !ignore_percentage && query_target_has_percentage_units(r->internal.qt);
 
     buffer_json_member_add_array(wb, key);
     for(size_t c = 0; c < r->d ; c++) {
@@ -1402,28 +1402,6 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
     }
 
     query_target_functions(wb, "functions", r);
-
-    buffer_json_member_add_object(wb, "db");
-    {
-        buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
-        buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
-        buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
-        buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
-
-        buffer_json_member_add_array(wb, "per_tier");
-        for(size_t tier = 0; tier < storage_tiers ; tier++) {
-            buffer_json_add_array_item_object(wb);
-            buffer_json_member_add_uint64(wb, "tier", tier);
-            buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
-            buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points);
-            buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every);
-            buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s);
-            buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s);
-            buffer_json_object_close(wb);
-        }
-        buffer_json_array_close(wb);
-    }
-    buffer_json_object_close(wb);
 }
 
 //static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) {
@@ -1512,6 +1490,37 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
     DATASOURCE_FORMAT format = qt->request.format;
     RRDR_OPTIONS options = qt->window.options;
 
+    buffer_json_member_add_object(wb, "db");
+    {
+        buffer_json_member_add_uint64(wb, "tiers", storage_tiers);
+        buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s);
+        buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s);
+        buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s);
+
+        query_target_combined_units_v2(wb, qt, r->internal.contexts, true);
+        buffer_json_member_add_object(wb, "dimensions");
+        {
+            rrdr_dimension_ids(wb, "ids", r, options);
+            rrdr_dimension_units_array_v2(wb, "units", r, options, true);
+            rrdr_dimension_query_points_statistics(wb, "sts", r, options, false);
+        }
+        buffer_json_object_close(wb); // dimensions
+
+        buffer_json_member_add_array(wb, "per_tier");
+        for(size_t tier = 0; tier < storage_tiers ; tier++) {
+            buffer_json_add_array_item_object(wb);
+            buffer_json_member_add_uint64(wb, "tier", tier);
+            buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries);
+            buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points);
+            buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every);
+            buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s);
+            buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s);
+            buffer_json_object_close(wb);
+        }
+        buffer_json_array_close(wb);
+    }
+    buffer_json_object_close(wb);
+
     buffer_json_member_add_object(wb, "view");
     {
         query_target_title(wb, qt, r->internal.contexts);
@@ -1536,18 +1545,17 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
         if(options & RRDR_OPTION_RETURN_RAW)
             buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
 
-        query_target_combined_units_v2(wb, qt, r->internal.contexts);
+        query_target_combined_units_v2(wb, qt, r->internal.contexts, false);
         query_target_combined_chart_type(wb, qt, r->internal.contexts);
         buffer_json_member_add_object(wb, "dimensions");
         {
             rrdr_grouped_by_array_v2(wb, "grouped_by", r, options);
             rrdr_dimension_ids(wb, "ids", r, options);
             rrdr_dimension_names(wb, "names", r, options);
-            rrdr_dimension_units_array_v2(wb, "units", r, options);
+            rrdr_dimension_units_array_v2(wb, "units", r, options, false);
             rrdr_dimension_priority_array_v2(wb, "priorities", r, options);
             rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options);
-            rrdr_dimension_query_points_statistics(wb, NULL, r, options, true);
-            rrdr_dimension_query_points_statistics(wb, "sts", r, options, false);
+            rrdr_dimension_query_points_statistics(wb, "sts", r, options, true);
             rrdr_json_group_by_labels(wb, "labels", r, options);
         }
         buffer_json_object_close(wb); // dimensions
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 5db5149d4d..16d8bfaf1d 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -394,6 +394,7 @@
                 "enum": [
                   "dimension",
                   "instance",
+                  "percentage-of-instance",
                   "label",
                   "node",
                   "context",
@@ -2994,6 +2995,50 @@
                     }
                   }
                 }
+              },
+              "units": {
+                "description": "The units of the database data\n",
+                "oneOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                ]
+              },
+              "dimensions": {
+                "type": "object",
+                "properties": {
+                  "ids": {
+                    "description": "An array with the dimension ids that uniquely identify the dimensions for this query. It is the same with `view.dimensions.ids`.\n",
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "units": {
+                    "description": "An array with the units each dimension has in the database (independent of group-by aggregation that may override the units).\n",
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "sts": {
+                    "description": "Statistics about the data collection points used for each dimension.\n",
+                    "oneOf": [
+                      {
+                        "$ref": "#/components/schemas/jsonwrap2_sts"
+                      },
+                      {
+                        "$ref": "#/components/schemas/jsonwrap2_sts_raw"
+                      }
+                    ]
+                  }
+                }
               }
             }
           },
@@ -3107,13 +3152,6 @@
                       "type": "string"
                     }
                   },
-                  "units": {
-                    "description": "An array with the units each dimension has.\n",
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  },
                   "priorities": {
                     "description": "An array with the relative priorities of the dimensions.\nNumbers may not be sequential or unique. The application is expected to order by this and then by name.\n",
                     "type": "array",
@@ -3128,88 +3166,23 @@
                       "type": "integer"
                     }
                   },
-                  "min": {
-                    "description": "An array of the minimum value of each dimension across the entire query.\n",
-                    "type": "array",
-                    "items": {
-                      "type": "number"
-                    }
-                  },
-                  "max": {
-                    "description": "An array of the maximum value of each dimension across the entire query.\n",
-                    "type": "array",
-                    "items": {
-                      "type": "number"
-                    }
-                  },
-                  "avg": {
-                    "description": "An array of the average value of each dimension across the entire query.\n",
+                  "units": {
+                    "description": "An array with the units each dimension has.\n",
                     "type": "array",
                     "items": {
-                      "type": "number"
+                      "type": "string"
                     }
                   },
                   "sts": {
-                    "description": "Statistics about the data collection points used for each dimension.\n",
-                    "type": "object",
-                    "properties": {
-                      "min": {
-                        "description": "An array with the minimum data collection value aggregated to each dimension.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
+                    "description": "Statistics about the view points for each dimension.\n",
+                    "oneOf": [
+                      {
+                        "$ref": "#/components/schemas/jsonwrap2_sts"
                       },
-                      "max": {
-                        "description": "An array with the maximum data collection value aggregated to each dimension.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "sum": {
-                        "description": "An array with the sum of all data collection values aggregated to each dimension.\nThis member exists only when option `raw` is given.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "cnt": {
-                        "description": "An array with the count of the data collection values aggregated to each dimension.\nThis member exists only when option `raw` is given.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "ars": {
-                        "description": "An array with the anomaly rate sum of all data collection values aggregated to each dimension.\nThis member exists only when option `raw` is given.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "avg": {
-                        "description": "An array with the average of all data collection values aggregated to each dimension.\nThis member exists only when option `raw` is not given. When option `raw` is given, the average can be calculated by dividing `sum` with `cnt`.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "arp": {
-                        "description": "An array with the average anomaly rate of all data collection values aggregated to each dimension.\nThis member exists only when option `raw` is not given. When option `raw` is given, the average can be calculated by dividing `ars` with `cnt`.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
-                      },
-                      "con": {
-                        "description": "An array with the contribution % of all data collection values aggregated to each dimension.\nThis member exists only when option `raw` is not given. When option `raw` is given, the contribution can be calculated by multiplying `ABS(sum)` with 100.0 and dividing it with the total of the ABS(sum) of all dimensions.\n",
-                        "type": "array",
-                        "items": {
-                          "type": "number"
-                        }
+                      {
+                        "$ref": "#/components/schemas/jsonwrap2_sts_raw"
                       }
-                    }
+                    ]
                   },
                   "labels": {
                     "description": "The labels associated with each dimension in the query.\nThis object is only available when the `group-by-labels` option is given to the query.\n",
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index 153692b80d..c25f0b7194 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -2192,6 +2192,35 @@ components:
                   last_entry:
                     description: |
                       The maximum unix epoch timestamp available across all metrics that used this tier. This reflects the newest timestamp of the tier's retention.
+            units:
+              description: |
+                The units of the database data
+              oneOf:
+                - type: string
+                - type: array
+                  items:
+                    type: string
+            dimensions:
+              type: object
+              properties:
+                ids:
+                  description: |
+                    An array with the dimension ids that uniquely identify the dimensions for this query. It is the same with `view.dimensions.ids`.
+                  type: array
+                  items:
+                    type: string
+                units:
+                  description: |
+                    An array with the units each dimension has in the database (independent of group-by aggregation that may override the units).
+                  type: array
+                  items:
+                    type: string
+                sts:
+                  description: |
+                    Statistics about the data collection points used for each dimension.
+                  oneOf:
+                    - $ref: "#/components/schemas/jsonwrap2_sts"
+                    - $ref: "#/components/schemas/jsonwrap2_sts_raw"
         view:
           type: object
           properties:
@@ -2295,12 +2324,6 @@ components:
                   type: array
                   items:
                     type: string
-                units:
-                  description: |
-                    An array with the units each dimension has.
-                  type: array
-                  items:
-                    type: string
                 priorities:
                   description: |
                     An array with the relative priorities of the dimensions.
@@ -2314,83 +2337,18 @@ components:
                   type: array
                   items:
                     type: integer
-                min:
-                  description: |
-                    An array of the minimum value of each dimension across the entire query.
-                  type: array
-                  items:
-                    type: number
-                max:
-                  description: |
-                    An array of the maximum value of each dimension across the entire query.
-                  type: array
-                  items:
-                    type: number
-                avg:
+                units:
                   description: |
-                    An array of the average value of each dimension across the entire query.
+                    An array with the units each dimension has.
                   type: array
                   items:
-                    type: number
+                    type: string
                 sts:
                   description: |
-                    Statistics about the data collection points used for each dimension.
-                  type: object
-                  properties:
-                    min:
-                      description: |
-                        An array with the minimum data collection value aggregated to each dimension.
-                      type: array
-                      items:
-                        type: number
-                    max:
-                      description: |
-                        An array with the maximum data collection value aggregated to each dimension.
-                      type: array
-                      items:
-                        type: number
-                    sum:
-                      description: |
-                        An array with the sum of all data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is given.
-                      type: array
-                      items:
-                        type: number
-                    cnt:
-                      description: |
-                        An array with the count of the data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is given.
-                      type: array
-                      items:
-                        type: number
-                    ars:
-                      description: |
-                        An array with the anomaly rate sum of all data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is given.
-                      type: array
-                      items:
-                        type: number
-                    avg:
-                      description: |
-                        An array with the average of all data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is not given. When option `raw` is given, the average can be calculated by dividing `sum` with `cnt`.
-                      type: array
-                      items:
-                        type: number
-                    arp:
-                      description: |
-                        An array with the average anomaly rate of all data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is not given. When option `raw` is given, the average can be calculated by dividing `ars` with `cnt`.
-                      type: array
-                      items:
-                        type: number
-                    con:
-                      description: |
-                        An array with the contribution % of all data collection values aggregated to each dimension.
-                        This member exists only when option `raw` is not given. When option `raw` is given, the contribution can be calculated by multiplying `ABS(sum)` with 100.0 and dividing it with the total of the ABS(sum) of all dimensions.
-                      type: array
-                      items:
-                        type: number
+                    Statistics about the view points for each dimension.
+                  oneOf:
+                    - $ref: "#/components/schemas/jsonwrap2_sts"
+                    - $ref: "#/components/schemas/jsonwrap2_sts_raw"
                 labels:
                   description: |
                     The labels associated with each dimension in the query.