From 66fd79777c90e737887d76409cf7bdaa6cb9dbd4 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 11 Apr 2023 17:32:22 +0300 Subject: /api/v2 part 9 (#14888) * /api/v2/weights now supports group-by * /api/v2/weights now follows the same principles for describing responses as /api/v2/data --- web/api/queries/weights.c | 625 ++++++++++++++++++++++++++++++++++------------ web/api/queries/weights.h | 7 + web/api/web_api.c | 20 ++ 3 files changed, 493 insertions(+), 159 deletions(-) (limited to 'web') diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index c50ed03a42..9dd42ccbab 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -303,116 +303,325 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w return total_dimensions; } +struct query_weights_data { + QUERY_WEIGHTS_REQUEST *qwr; + + SIMPLE_PATTERN *scope_nodes_sp; + SIMPLE_PATTERN *scope_contexts_sp; + SIMPLE_PATTERN *nodes_sp; + SIMPLE_PATTERN *contexts_sp; + SIMPLE_PATTERN *instances_sp; + SIMPLE_PATTERN *dimensions_sp; + SIMPLE_PATTERN *labels_sp; + SIMPLE_PATTERN *alerts_sp; + + usec_t timeout_us; + bool timed_out; + bool interrupted; + + struct query_timings timings; + + size_t examined_dimensions; + bool register_zero; + + DICTIONARY *results; + WEIGHTS_STATS stats; + + uint32_t shifts; + + struct query_versions versions; +}; + +#define AGGREGATED_WEIGHT_EMPTY (struct aggregated_weight) { \ + .min = NAN, \ + .max = NAN, \ + .sum = NAN, \ + .count = 0, \ + .hsp = STORAGE_POINT_UNSET, \ + .bsp = STORAGE_POINT_UNSET, \ +} + +#define merge_into_aw(aw, t) do { \ + if(!(aw).count) { \ + (aw).count = 1; \ + (aw).min = (aw).max = (aw).sum = (t)->value; \ + (aw).hsp = (t)->highlighted; \ + if(baseline) \ + (aw).bsp = (t)->baseline; \ + } \ + else { \ + (aw).count++; \ + (aw).sum += (t)->value; \ + if((t)->value < (aw).min) \ + (aw).min = (t)->value; \ + if((t)->value > (aw).max) \ + (aw).max = (t)->value; \ + storage_point_merge_to((aw).hsp, (t)->highlighted); \ + if(baseline) \ + storage_point_merge_to((aw).bsp, (t)->baseline); \ + } \ +} while(0) + +static void results_header_to_json_v2(DICTIONARY *results __maybe_unused, BUFFER *wb, struct query_weights_data *qwd, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions __maybe_unused, usec_t duration, + WEIGHTS_STATS *stats, bool group_by) { + + buffer_json_member_add_object(wb, "request"); + buffer_json_member_add_string(wb, "method", weights_method_to_string(method)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); + + buffer_json_member_add_object(wb, "scope"); + buffer_json_member_add_string(wb, "scope_nodes", qwd->qwr->scope_nodes ? qwd->qwr->scope_nodes : "*"); + buffer_json_member_add_string(wb, "scope_contexts", qwd->qwr->scope_contexts ? qwd->qwr->scope_contexts : "*"); + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "selectors"); + buffer_json_member_add_string(wb, "nodes", qwd->qwr->nodes ? qwd->qwr->nodes : "*"); + buffer_json_member_add_string(wb, "contexts", qwd->qwr->contexts ? qwd->qwr->contexts : "*"); + buffer_json_member_add_string(wb, "instances", qwd->qwr->instances ? qwd->qwr->instances : "*"); + buffer_json_member_add_string(wb, "dimensions", qwd->qwr->dimensions ? qwd->qwr->dimensions : "*"); + buffer_json_member_add_string(wb, "labels", qwd->qwr->labels ? qwd->qwr->labels : "*"); + buffer_json_member_add_string(wb, "alerts", qwd->qwr->alerts ? qwd->qwr->alerts : "*"); + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", qwd->qwr->after); + buffer_json_member_add_time_t(wb, "before", qwd->qwr->before); + buffer_json_member_add_uint64(wb, "points", qwd->qwr->points); + if(qwd->qwr->options & RRDR_OPTION_SELECTED_TIER) + buffer_json_member_add_uint64(wb, "tier", qwd->qwr->tier); + else + buffer_json_member_add_string(wb, "tier", NULL); + buffer_json_object_close(wb); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + buffer_json_member_add_object(wb, "baseline"); + buffer_json_member_add_time_t(wb, "baseline_after", qwd->qwr->baseline_after); + buffer_json_member_add_time_t(wb, "baseline_before", qwd->qwr->baseline_before); + buffer_json_object_close(wb); + } + + buffer_json_member_add_object(wb, "aggregations"); + buffer_json_member_add_object(wb, "time"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qwd->qwr->time_group_method)); + buffer_json_member_add_string(wb, "time_group_options", qwd->qwr->time_group_options); + buffer_json_object_close(wb); // time + + buffer_json_member_add_array(wb, "metrics"); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_array(wb, "group_by"); + buffer_json_group_by_to_array(wb, qwd->qwr->group_by.group_by); + buffer_json_array_close(wb); + +// buffer_json_member_add_array(wb, "group_by_label"); +// buffer_json_array_close(wb); + + buffer_json_member_add_string(wb, "aggregation", group_by_aggregate_function_to_string(qwd->qwr->group_by.aggregation)); + } + buffer_json_object_close(wb); // 1st group by + buffer_json_array_close(wb); // array + buffer_json_object_close(wb); // aggregations + + buffer_json_member_add_uint64(wb, "timeout", qwd->qwr->timeout_ms); + buffer_json_object_close(wb); // request + + buffer_json_member_add_object(wb, "view"); + buffer_json_member_add_string(wb, "format", (group_by)?"grouped":"full"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(group)); + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", after); + buffer_json_member_add_time_t(wb, "before", before); + buffer_json_member_add_time_t(wb, "duration", before - after); + buffer_json_member_add_uint64(wb, "points", points); + buffer_json_object_close(wb); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + buffer_json_member_add_object(wb, "baseline"); + buffer_json_member_add_time_t(wb, "baseline_after", baseline_after); + buffer_json_member_add_time_t(wb, "baseline_before", baseline_before); + buffer_json_member_add_time_t(wb, "baseline_duration", baseline_before - baseline_after); + buffer_json_member_add_uint64(wb, "baseline_points", points << shifts); + buffer_json_object_close(wb); + } + + buffer_json_object_close(wb); // view + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_double(wb, "query_time_ms", (double) duration / (double) USEC_PER_MS); + buffer_json_member_add_uint64(wb, "db_queries", stats->db_queries); + buffer_json_member_add_uint64(wb, "query_result_points", stats->result_points); + buffer_json_member_add_uint64(wb, "binary_searches", stats->binary_searches); + buffer_json_member_add_uint64(wb, "db_points_read", stats->db_points); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + { + for (size_t tier = 0; tier < storage_tiers; tier++) + buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); // db +} + typedef enum { WPT_DIMENSION = 0, WPT_INSTANCE = 1, WPT_CONTEXT = 2, WPT_NODE = 3, + WPT_GROUP = 4, } WEIGHTS_POINT_TYPE; -static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, NETDATA_DOUBLE weight, STORAGE_POINT *highlighted_sp, STORAGE_POINT *baseline_sp, RRDR_OPTIONS options __maybe_unused, bool baseline) { - buffer_json_add_array_item_array(wb); +struct aggregated_weight { + const char *name; + NETDATA_DOUBLE min; + NETDATA_DOUBLE max; + NETDATA_DOUBLE sum; + size_t count; + STORAGE_POINT hsp; + STORAGE_POINT bsp; +}; - buffer_json_add_array_item_uint64(wb, type); // "type" - buffer_json_add_array_item_int64(wb, ni); - if(type != WPT_NODE) { - buffer_json_add_array_item_int64(wb, ci); - if(type != WPT_CONTEXT) { - buffer_json_add_array_item_int64(wb, ii); - if(type != WPT_INSTANCE) - buffer_json_add_array_item_int64(wb, di); - else +static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, struct aggregated_weight *aw, RRDR_OPTIONS options __maybe_unused, bool baseline) { + if(type != WPT_GROUP) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_uint64(wb, type); // "type" + buffer_json_add_array_item_int64(wb, ni); + if (type != WPT_NODE) { + buffer_json_add_array_item_int64(wb, ci); + if (type != WPT_CONTEXT) { + buffer_json_add_array_item_int64(wb, ii); + if (type != WPT_INSTANCE) + buffer_json_add_array_item_int64(wb, di); + else + buffer_json_add_array_item_string(wb, NULL); + } + else { + buffer_json_add_array_item_string(wb, NULL); buffer_json_add_array_item_string(wb, NULL); - } else { + } + } + else { + buffer_json_add_array_item_string(wb, NULL); buffer_json_add_array_item_string(wb, NULL); buffer_json_add_array_item_string(wb, NULL); } + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "weight" } else { - buffer_json_add_array_item_string(wb, NULL); - buffer_json_add_array_item_string(wb, NULL); - buffer_json_add_array_item_string(wb, NULL); + buffer_json_member_add_array(wb, "v"); + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_double(wb, aw->min); // "min" + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->max); // "max" + buffer_json_add_array_item_double(wb, aw->sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->count); // "count" + buffer_json_array_close(wb); } - buffer_json_add_array_item_double(wb, weight); // "weight" buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_double(wb, highlighted_sp->min); // "min" - buffer_json_add_array_item_double(wb, (highlighted_sp->count) ? highlighted_sp->sum / (NETDATA_DOUBLE) highlighted_sp->count : 0.0); // "avg" - buffer_json_add_array_item_double(wb, highlighted_sp->max); // "max" - buffer_json_add_array_item_double(wb, highlighted_sp->sum); // "sum" - buffer_json_add_array_item_uint64(wb, highlighted_sp->count); // "count" - buffer_json_add_array_item_uint64(wb, highlighted_sp->anomaly_count); // "anomaly_count" + buffer_json_add_array_item_double(wb, aw->hsp.min); // "min" + buffer_json_add_array_item_double(wb, (aw->hsp.count) ? aw->hsp.sum / (NETDATA_DOUBLE) aw->hsp.count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->hsp.max); // "max" + buffer_json_add_array_item_double(wb, aw->hsp.sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->hsp.count); // "count" + buffer_json_add_array_item_uint64(wb, aw->hsp.anomaly_count); // "anomaly_count" buffer_json_array_close(wb); if(baseline) { buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_double(wb, baseline_sp->min); // "min" - buffer_json_add_array_item_double(wb, (baseline_sp->count) ? baseline_sp->sum / (NETDATA_DOUBLE) baseline_sp->count : 0.0); // "avg" - buffer_json_add_array_item_double(wb, baseline_sp->max); // "max" - buffer_json_add_array_item_double(wb, baseline_sp->sum); // "sum" - buffer_json_add_array_item_uint64(wb, baseline_sp->count); // "count" - buffer_json_add_array_item_uint64(wb, baseline_sp->anomaly_count); // "anomaly_count" + buffer_json_add_array_item_double(wb, aw->bsp.min); // "min" + buffer_json_add_array_item_double(wb, (aw->bsp.count) ? aw->bsp.sum / (NETDATA_DOUBLE) aw->bsp.count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->bsp.max); // "max" + buffer_json_add_array_item_double(wb, aw->bsp.sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->bsp.count); // "count" + buffer_json_add_array_item_uint64(wb, aw->bsp.anomaly_count); // "anomaly_count" buffer_json_array_close(wb); } - buffer_json_array_close(wb); // key + buffer_json_array_close(wb); } -static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline) { +static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline, bool group_by) { buffer_json_member_add_object(wb, key); // schema buffer_json_member_add_string(wb, "type", "array"); buffer_json_member_add_array(wb, "items"); - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "name", "row_type"); - buffer_json_member_add_string(wb, "type", "integer"); - buffer_json_member_add_array(wb, "value"); - buffer_json_add_array_item_string(wb, "dimension"); - buffer_json_add_array_item_string(wb, "instance"); - buffer_json_add_array_item_string(wb, "context"); - buffer_json_add_array_item_string(wb, "node"); - buffer_json_array_close(wb); - buffer_json_object_close(wb); - - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", "ni"); - buffer_json_member_add_string(wb, "type", "integer"); - buffer_json_member_add_string(wb, "dictionary", "nodes"); + if(group_by) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "weight"); + buffer_json_member_add_string(wb, "type", "array"); + buffer_json_member_add_array(wb, "labels"); + { + buffer_json_add_array_item_string(wb, "min"); + buffer_json_add_array_item_string(wb, "avg"); + buffer_json_add_array_item_string(wb, "max"); + buffer_json_add_array_item_string(wb, "sum"); + buffer_json_add_array_item_string(wb, "count"); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); } - buffer_json_object_close(wb); - - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", "ci"); + else { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "name", "row_type"); buffer_json_member_add_string(wb, "type", "integer"); - buffer_json_member_add_string(wb, "dictionary", "contexts"); - } - buffer_json_object_close(wb); + buffer_json_member_add_array(wb, "value"); + buffer_json_add_array_item_string(wb, "dimension"); + buffer_json_add_array_item_string(wb, "instance"); + buffer_json_add_array_item_string(wb, "context"); + buffer_json_add_array_item_string(wb, "node"); + buffer_json_array_close(wb); + buffer_json_object_close(wb); - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", "ii"); - buffer_json_member_add_string(wb, "type", "integer"); - buffer_json_member_add_string(wb, "dictionary", "instances"); - } - buffer_json_object_close(wb); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ni"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "nodes"); + } + buffer_json_object_close(wb); - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", "di"); - buffer_json_member_add_string(wb, "type", "integer"); - buffer_json_member_add_string(wb, "dictionary", "dimensions"); - } - buffer_json_object_close(wb); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ci"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "contexts"); + } + buffer_json_object_close(wb); - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", "weight"); - buffer_json_member_add_string(wb, "type", "number"); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ii"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "instances"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "di"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "dimensions"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "weight"); + buffer_json_member_add_string(wb, "type", "number"); + } + buffer_json_object_close(wb); } - buffer_json_object_close(wb); buffer_json_add_array_item_object(wb); { @@ -518,53 +727,26 @@ static inline ssize_t dict_unique_id_name_add(DICTIONARY *dict, const char *id, return (ssize_t)dun->i; } -struct query_weights_data { - QUERY_WEIGHTS_REQUEST *qwr; - - SIMPLE_PATTERN *scope_nodes_sp; - SIMPLE_PATTERN *scope_contexts_sp; - SIMPLE_PATTERN *nodes_sp; - SIMPLE_PATTERN *contexts_sp; - SIMPLE_PATTERN *instances_sp; - SIMPLE_PATTERN *dimensions_sp; - SIMPLE_PATTERN *labels_sp; - SIMPLE_PATTERN *alerts_sp; - - usec_t timeout_us; - bool timed_out; - bool interrupted; - - struct query_timings timings; - - size_t examined_dimensions; - bool register_zero; - - DICTIONARY *results; - WEIGHTS_STATS stats; - - uint32_t shifts; - - struct query_versions versions; -}; - -static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER *wb, - time_t after, time_t before, - time_t baseline_after, time_t baseline_before, - size_t points, WEIGHTS_METHOD method, - RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, - size_t examined_dimensions, struct query_weights_data *qwd, - WEIGHTS_STATS *stats, - struct query_versions *versions) { +static size_t registered_results_to_json_multinode_no_group_by( + DICTIONARY *results, BUFFER *wb, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, struct query_weights_data *qwd, + WEIGHTS_STATS *stats, + struct query_versions *versions) { buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); buffer_json_member_add_uint64(wb, "api", 2); - results_header_to_json(results, wb, after, before, baseline_after, baseline_before, - points, method, group, options, shifts, examined_dimensions, qwd->timings.executed_ut - qwd->timings.received_ut, stats); + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, + qwd->timings.executed_ut - qwd->timings.received_ut, stats, false); version_hashes_api_v2(wb, versions); bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME; - multinode_data_schema(wb, options, "schema", baseline); + multinode_data_schema(wb, options, "schema", baseline, false); DICTIONARY *dict_nodes = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_node)); DICTIONARY *dict_contexts = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_name_units)); @@ -573,10 +755,7 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER * buffer_json_member_add_array(wb, "result"); - size_t total_dimensions = 0, node_dims = 0, context_dims = 0, instance_dims = 0; - NETDATA_DOUBLE context_total_weight = 0.0, instance_total_weight = 0.0, node_total_weight = 0.0; - STORAGE_POINT context_hsp = STORAGE_POINT_UNSET, instance_hsp = STORAGE_POINT_UNSET, node_hsp = STORAGE_POINT_UNSET; - STORAGE_POINT context_bsp = STORAGE_POINT_UNSET, instance_bsp = STORAGE_POINT_UNSET, node_bsp = STORAGE_POINT_UNSET; + struct aggregated_weight node_aw = AGGREGATED_WEIGHT_EMPTY, context_aw = AGGREGATED_WEIGHT_EMPTY, instance_aw = AGGREGATED_WEIGHT_EMPTY; struct register_result *t; RRDHOST *last_host = NULL; RRDCONTEXT_ACQUIRED *last_rca = NULL; @@ -584,34 +763,28 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER * struct dict_unique_node *node_dun = NULL; ssize_t di = -1, ii = -1, ci = -1, ni = -1; ssize_t di_max = 0, ii_max = 0, ci_max = 0, ni_max = 0; + size_t total_dimensions = 0; dfe_start_read(results, t) { // close instance if(t->ria != last_ria && last_ria) { - storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline); - + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline); last_ria = NULL; - instance_dims = 0; - instance_total_weight = 0.0; - instance_hsp = instance_bsp = STORAGE_POINT_UNSET; + instance_aw = AGGREGATED_WEIGHT_EMPTY; } // close context if(t->rca != last_rca && last_rca) { - storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline); + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline); last_rca = NULL; - context_dims = 0; - context_total_weight = 0.0; - context_hsp = context_bsp = STORAGE_POINT_UNSET; + context_aw = AGGREGATED_WEIGHT_EMPTY; } // close node if(t->host != last_host && last_host) { - storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline); + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline); last_host = NULL; - node_dims = 0; - node_total_weight = 0.0; - node_hsp = node_bsp = STORAGE_POINT_UNSET; + node_aw = AGGREGATED_WEIGHT_EMPTY; } // open node @@ -635,42 +808,37 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER * } di = dict_unique_id_name_add(dict_dimensions, rrdmetric_acquired_id(t->rma), rrdmetric_acquired_name(t->rma), &di_max); - storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, t->value, &t->highlighted, &t->baseline, options, baseline); - instance_total_weight += t->value; - context_total_weight += t->value; - node_total_weight += t->value; + struct aggregated_weight aw = { + .min = t->value, + .max = t->value, + .sum = t->value, + .count = 1, + .hsp = t->highlighted, + .bsp = t->baseline, + }; + storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, &aw, options, baseline); - node_dun->duration_ut += t->duration_ut; - - storage_point_merge_to(instance_hsp, t->highlighted); - storage_point_merge_to(context_hsp, t->highlighted); - storage_point_merge_to(node_hsp, t->highlighted); + merge_into_aw(instance_aw, t); + merge_into_aw(context_aw, t); + merge_into_aw(node_aw, t); - if(baseline) { - storage_point_merge_to(instance_bsp, t->baseline); - storage_point_merge_to(context_bsp, t->baseline); - storage_point_merge_to(node_bsp, t->baseline); - } - - instance_dims++; - context_dims++; - node_dims++; + node_dun->duration_ut += t->duration_ut; total_dimensions++; } dfe_done(t); // close instance if(last_ria) - storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, instance_total_weight / (double) instance_dims, &instance_hsp, &instance_bsp, options, baseline); + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline); // close context if(last_rca) - storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, context_total_weight / (double) context_dims, &context_hsp, &instance_bsp, options, baseline); + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline); // close node if(last_host) - storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, node_total_weight / (double) node_dims, &node_hsp, &node_bsp, options, baseline); + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline); buffer_json_array_close(wb); // points @@ -746,6 +914,131 @@ static size_t registered_results_to_json_multinode(DICTIONARY *results, BUFFER * return total_dimensions; } +static size_t registered_results_to_json_multinode_group_by( + DICTIONARY *results, BUFFER *wb, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, struct query_weights_data *qwd, + WEIGHTS_STATS *stats, + struct query_versions *versions) { + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_member_add_uint64(wb, "api", 2); + + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, + qwd->timings.executed_ut - qwd->timings.received_ut, stats, true); + + version_hashes_api_v2(wb, versions); + + bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME; + multinode_data_schema(wb, options, "v_schema", baseline, true); + + DICTIONARY *group_by = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(struct aggregated_weight)); + + struct register_result *t; + size_t total_dimensions = 0; + BUFFER *key = buffer_create(0, NULL); + BUFFER *name = buffer_create(0, NULL); + dfe_start_read(results, t) { + + buffer_flush(key); + buffer_flush(name); + + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_DIMENSION) { + buffer_strcat(key, rrdmetric_acquired_name(t->rma)); + buffer_strcat(name, rrdmetric_acquired_name(t->rma)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_INSTANCE) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdinstance_acquired_id(t->ria)); + buffer_strcat(name, rrdinstance_acquired_name(t->ria)); + + if(!(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE)) { + buffer_fast_strcat(key, "@", 1); + buffer_fast_strcat(name, "@", 1); + buffer_strcat(key, t->host->machine_guid); + buffer_strcat(name, rrdhost_hostname(t->host)); + } + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, t->host->machine_guid); + buffer_strcat(name, rrdhost_hostname(t->host)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_CONTEXT) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdcontext_acquired_id(t->rca)); + buffer_strcat(name, rrdcontext_acquired_id(t->rca)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_UNITS) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdcontext_acquired_units(t->rca)); + buffer_strcat(name, rrdcontext_acquired_units(t->rca)); + } + + struct aggregated_weight *aw = dictionary_set(group_by, buffer_tostring(key), NULL, sizeof(struct aggregated_weight)); + if(!aw->name) { + aw->name = strdupz(buffer_tostring(name)); + aw->min = aw->max = aw->sum = t->value; + aw->count = 1; + aw->hsp = t->highlighted; + aw->bsp = t->baseline; + } + else + merge_into_aw(*aw, t); + + total_dimensions++; + } + dfe_done(t); + buffer_free(key); key = NULL; + buffer_free(name); name = NULL; + + struct aggregated_weight *aw; + buffer_json_member_add_array(wb, "result"); + dfe_start_read(group_by, aw) { + const char *k = aw_dfe.name; + const char *n = aw->name; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", k); + buffer_json_member_add_string(wb, "nm", n); + storage_point_to_json(wb, WPT_GROUP, 0, 0, 0, 0, aw, options, baseline); + buffer_json_object_close(wb); + + freez((void *)aw->name); + } + dfe_done(aw); + buffer_json_array_close(wb); // result + + buffer_json_agents_array_v2(wb, &qwd->timings, 0); + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions); + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions); + buffer_json_finalize(wb); + + dictionary_destroy(group_by); + + return total_dimensions; +} + // ---------------------------------------------------------------------------- // KS2 algorithm functions @@ -1616,14 +1909,28 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) { default: case WEIGHTS_FORMAT_MULTINODE: - added_dimensions = - registered_results_to_json_multinode( - qwd.results, wb, - qwr->after, qwr->before, - qwr->baseline_after, qwr->baseline_before, - qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, - qwd.examined_dimensions, - &qwd, &qwd.stats, &qwd.versions); + // we don't support these groupings in weights + qwr->group_by.group_by &= ~(RRDR_GROUP_BY_LABEL|RRDR_GROUP_BY_SELECTED|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE); + if(qwr->group_by.group_by == RRDR_GROUP_BY_NONE) { + added_dimensions = + registered_results_to_json_multinode_no_group_by( + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + &qwd, &qwd.stats, &qwd.versions); + } + else { + added_dimensions = + registered_results_to_json_multinode_group_by( + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + &qwd, &qwd.stats, &qwd.versions); + } break; } diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h index aa9e643f17..66bea6ab2f 100644 --- a/web/api/queries/weights.h +++ b/web/api/queries/weights.h @@ -35,6 +35,13 @@ typedef struct query_weights_request { const char *dimensions; const char *labels; const char *alerts; + + struct { + RRDR_GROUP_BY group_by; + char *group_by_label; + RRDR_GROUP_BY_FUNCTION aggregation; + } group_by; + WEIGHTS_METHOD method; WEIGHTS_FORMAT format; RRDR_TIME_GROUPING time_group_method; diff --git a/web/api/web_api.c b/web/api/web_api.c index 900ae43dcc..7dd35717d4 100644 --- a/web/api/web_api.c +++ b/web/api/web_api.c @@ -71,6 +71,12 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur const char *time_group_options = NULL, *scope_contexts = NULL, *scope_nodes = NULL, *contexts = NULL, *nodes = NULL, *instances = NULL, *dimensions = NULL, *labels = NULL, *alerts = NULL; + struct group_by_pass group_by = { + .group_by = RRDR_GROUP_BY_NONE, + .group_by_label = NULL, + .aggregation = RRDR_GROUP_BY_FUNCTION_AVERAGE, + }; + while (url) { char *value = mystrsep(&url, "&"); if (!value || !*value) @@ -123,6 +129,15 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur else if(api_version >= 2 && !strcmp(name, "dimensions")) dimensions = value; else if(api_version >= 2 && !strcmp(name, "labels")) labels = value; else if(api_version >= 2 && !strcmp(name, "alerts")) alerts = value; + else if(api_version >= 2 && (!strcmp(name, "group_by") || !strcmp(name, "group_by[0]"))) { + group_by.group_by = group_by_parse(value); + } + else if(api_version >= 2 && (!strcmp(name, "group_by_label") || !strcmp(name, "group_by_label[0]"))) { + group_by.group_by_label = value; + } + else if(api_version >= 2 && (!strcmp(name, "aggregation") || !strcmp(name, "aggregation[0]"))) { + group_by.aggregation = group_by_aggregate_function_parse(value); + } else if(!strcmp(name, "tier")) { tier = str2ul(value); @@ -161,6 +176,11 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur .dimensions = dimensions, .labels = labels, .alerts = alerts, + .group_by = { + .group_by = group_by.group_by, + .group_by_label = group_by.group_by_label, + .aggregation = group_by.aggregation, + }, .method = method, .format = format, .time_group_method = time_group_method, -- cgit v1.2.3