From 204dd9ae272445d13f308badb07e99675fa34892 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Fri, 7 Apr 2023 21:25:01 +0300 Subject: Boost dbengine (#14832) * configure extent cache size * workers can now execute up to 10 jobs in a run, boosting query prep and extent reads * fix dispatched and executing counters * boost to the max * increase libuv worker threads * query prep always get more prio than extent reads; stop processing in batch when dbengine is queue is critical * fix accounting of query prep * inlining of time-grouping functions, to speed up queries with billions of points * make switching based on a local const variable * print one pending contexts loading message per iteration * inlined store engine query API * inlined storage engine data collection api * inlined all storage engine query ops * eliminate and inline data collection ops * simplified query group-by * more error handling * optimized partial trimming of group-by queries * preparative work to support multiple passes of group-by * more preparative work to support multiple passes of group-by (accepts multiple group-by params) * unified query timings * unified query timings - weights endpoint * query target is no longer a static thread variable - there is a list of cached query targets, each of which of freed every 1000 queries * fix query memory accounting * added summary.dimension[].pri and sorted summary.dimensions based on priority and then name * limit max ACLK WEB response size to 30MB * the response type should be text/plain * more preparative work for multiple group-by passes * create functions for generating group by keys, ids and names * multiple group-by passes are now supported * parse group-by options array also with an index * implemented percentage-of-instance group by function * family is now merged in multi-node contexts * prevent uninitialized use --- web/api/formatters/json_wrapper.c | 208 ++- web/api/formatters/rrd2json.c | 8 - web/api/formatters/rrd2json.h | 4 - web/api/formatters/value/value.c | 4 +- web/api/netdata-swagger.yaml | 1 + web/api/queries/average/average.c | 52 - web/api/queries/average/average.h | 57 +- web/api/queries/countif/countif.c | 129 -- web/api/queries/countif/countif.h | 143 +- web/api/queries/des/des.c | 129 -- web/api/queries/des/des.h | 133 +- web/api/queries/incremental_sum/incremental_sum.c | 59 - web/api/queries/incremental_sum/incremental_sum.h | 64 +- web/api/queries/max/max.c | 50 - web/api/queries/max/max.h | 54 +- web/api/queries/median/median.c | 134 -- web/api/queries/median/median.h | 146 +- web/api/queries/min/min.c | 50 - web/api/queries/min/min.h | 54 +- web/api/queries/percentile/percentile.c | 163 -- web/api/queries/percentile/percentile.h | 175 ++- web/api/queries/query.c | 1688 ++++++++++++--------- web/api/queries/query.h | 24 +- web/api/queries/rrdr.c | 4 +- web/api/queries/rrdr.h | 17 +- web/api/queries/ses/ses.c | 82 - web/api/queries/ses/ses.h | 87 +- web/api/queries/stddev/stddev.c | 112 -- web/api/queries/stddev/stddev.h | 118 +- web/api/queries/sum/sum.c | 46 - web/api/queries/sum/sum.h | 51 +- web/api/queries/trimmed_mean/trimmed_mean.c | 159 -- web/api/queries/trimmed_mean/trimmed_mean.h | 171 ++- web/api/queries/weights.c | 97 +- web/api/web_api_v2.c | 85 +- web/server/web_client.h | 1 + 36 files changed, 2458 insertions(+), 2101 deletions(-) (limited to 'web') diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index b8f47ea57d..b19ce2590a 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -368,17 +368,60 @@ static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, cons buffer_json_array_close(wb); } +struct dimensions_sorted_walkthrough_data { + BUFFER *wb; + struct summary_total_counts *totals; + QUERY_TARGET *qt; +}; + +struct dimensions_sorted_entry { + const char *id; + const char *name; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; + uint32_t priority; +}; + +static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct dimensions_sorted_walkthrough_data *sdwd = data; + BUFFER *wb = sdwd->wb; + struct summary_total_counts *totals = sdwd->totals; + QUERY_TARGET *qt = sdwd->qt; + struct dimensions_sorted_entry *z = value; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z->id); + if (z->id != z->name && z->name) + buffer_json_member_add_string(wb, "nm", z->name); + + query_target_metric_counts(wb, &z->metrics); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_member_add_uint64(wb, "pri", z->priority); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &z->metrics); + + return 1; +} + +int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) { + struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1); + struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2); + + if(z1->priority == z2->priority) + return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2)); + else if(z1->priority < z2->priority) + return -1; + else + return 1; +} + static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) { - char name[RRD_ID_LENGTH_MAX * 2 + 2]; + char buf[RRD_ID_LENGTH_MAX * 2 + 2]; buffer_json_member_add_array(wb, key); DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - struct { - const char *id; - const char *name; - STORAGE_POINT query_points; - QUERY_METRICS_COUNTS metrics; - } *z; + struct dimensions_sorted_entry *z; size_t q = 0; for (long c = 0; c < (long) qt->dimensions.used; c++) { QUERY_DIMENSION * qd = query_dimension(qt, c); @@ -392,23 +435,31 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co qm = tqm; } + const char *key, *id, *name; + if(v2) { - z = dictionary_set(dict, rrdmetric_acquired_name(rma), NULL, sizeof(*z)); - if(!z->id) - z->id = rrdmetric_acquired_name(rma); - if(!z->name) - z->name = rrdmetric_acquired_name(rma); + key = rrdmetric_acquired_name(rma); + id = key; + name = key; } else { - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", rrdmetric_acquired_id(rma), rrdmetric_acquired_name(rma)); + key = buf; + id = rrdmetric_acquired_id(rma); + name = rrdmetric_acquired_name(rma); + } - z = dictionary_set(dict, name, NULL, sizeof(*z)); - if (!z->id) - z->id = rrdmetric_acquired_id(rma); - if (!z->name) - z->name = rrdmetric_acquired_name(rma); + z = dictionary_set(dict, key, NULL, sizeof(*z)); + if(!z->id) { + z->id = id; + z->name = name; + z->priority = qd->priority; + } + else { + if(qd->priority < z->priority) + z->priority = qd->priority; } if(qm) { @@ -423,27 +474,26 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co else z->metrics.excluded++; } - dfe_start_read(dict, z) { - if(v2) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", z->id); - if(z->id != z->name) - buffer_json_member_add_string(wb, "nm", z->name); - - query_target_metric_counts(wb, &z->metrics); - query_target_points_statistics(wb, qt, &z->query_points); - buffer_json_object_close(wb); - aggregate_into_summary_totals(totals, &z->metrics); - } - else { - buffer_json_add_array_item_array(wb); - buffer_json_add_array_item_string(wb, z->id); - buffer_json_add_array_item_string(wb, z->name); - buffer_json_array_close(wb); - } - } - dfe_done(z); + if(v2) { + struct dimensions_sorted_walkthrough_data t = { + .wb = wb, + .totals = totals, + .qt = qt, + }; + dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb, + &t, dimensions_sorted_compar); + } + else { + // v1 + dfe_start_read(dict, z) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, z->id); + buffer_json_add_array_item_string(wb, z->name); + buffer_json_array_close(wb); + } + dfe_done(z); + } dictionary_destroy(dict); buffer_json_array_close(wb); } @@ -805,18 +855,6 @@ static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char buffer_json_object_close(wb); } -static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) { - QUERY_TARGET *qt = r->internal.qt; - - qt->timings.finished_ut = now_monotonic_usec(); - buffer_json_member_add_object(wb, key); - buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(qt->timings.preprocessed_ut - qt->timings.received_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(qt->timings.executed_ut - qt->timings.preprocessed_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.executed_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.received_ut) / USEC_PER_MS); - buffer_json_object_close(wb); -} - void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) { QUERY_TARGET *qt = r->internal.qt; DATASOURCE_FORMAT format = qt->request.format; @@ -948,35 +986,50 @@ static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_ buffer_json_member_add_array(wb, key); - if(qt->request.group_by & RRDR_GROUP_BY_SELECTED) + // find the deeper group-by + ssize_t g = 0; + for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; + } + + if(g > 0) + g--; + + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; + + if(group_by & RRDR_GROUP_BY_SELECTED) buffer_json_add_array_item_string(wb, "selected"); + else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + buffer_json_add_array_item_string(wb, "percentage-of-instance"); + else { - if(qt->request.group_by & RRDR_GROUP_BY_DIMENSION) + if(group_by & RRDR_GROUP_BY_DIMENSION) buffer_json_add_array_item_string(wb, "dimension"); - if(qt->request.group_by & RRDR_GROUP_BY_INSTANCE) + if(group_by & RRDR_GROUP_BY_INSTANCE) buffer_json_add_array_item_string(wb, "instance"); - if(qt->request.group_by & RRDR_GROUP_BY_LABEL) { + if(group_by & RRDR_GROUP_BY_LABEL) { BUFFER *b = buffer_create(0, NULL); - for (size_t l = 0; l < qt->group_by.used; l++) { + for (size_t l = 0; l < qt->group_by[g].used; l++) { buffer_flush(b); buffer_fast_strcat(b, "label:", 6); - buffer_strcat(b, qt->group_by.label_keys[l]); + buffer_strcat(b, qt->group_by[g].label_keys[l]); buffer_json_add_array_item_string(wb, buffer_tostring(b)); } buffer_free(b); } - if(qt->request.group_by & RRDR_GROUP_BY_NODE) + if(group_by & RRDR_GROUP_BY_NODE) buffer_json_add_array_item_string(wb, "node"); - if(qt->request.group_by & RRDR_GROUP_BY_CONTEXT) + if(group_by & RRDR_GROUP_BY_CONTEXT) buffer_json_add_array_item_string(wb, "context"); - if(qt->request.group_by & RRDR_GROUP_BY_UNITS) + if(group_by & RRDR_GROUP_BY_UNITS) buffer_json_add_array_item_string(wb, "units"); } @@ -1237,7 +1290,6 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) { buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY); buffer_json_member_add_uint64(wb, "api", 2); - buffer_json_agents_array_v2(wb, 0); if(options & RRDR_OPTION_DEBUG) { buffer_json_member_add_string(wb, "id", qt->id); @@ -1284,21 +1336,28 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) { buffer_json_member_add_string(wb, "time_resampling", NULL); buffer_json_object_close(wb); // time - buffer_json_member_add_object(wb, "metrics"); + buffer_json_member_add_array(wb, "metrics"); + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; - buffer_json_member_add_array(wb, "group_by"); - buffer_json_group_by_to_array(wb, qt->request.group_by); - buffer_json_array_close(wb); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_array(wb, "group_by"); + buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by); + buffer_json_array_close(wb); - buffer_json_member_add_array(wb, "group_by_label"); - for(size_t l = 0; l < qt->group_by.used ;l++) - buffer_json_add_array_item_string(wb, qt->group_by.label_keys[l]); - buffer_json_array_close(wb); + buffer_json_member_add_array(wb, "group_by_label"); + for (size_t l = 0; l < qt->group_by[g].used; l++) + buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]); + buffer_json_array_close(wb); - buffer_json_member_add_string(wb, "aggregation", - group_by_aggregate_function_to_string( - qt->request.group_by_aggregate_function)); - buffer_json_object_close(wb); // dimensions + buffer_json_member_add_string( + wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation)); + } + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); // group_by } buffer_json_object_close(wb); // aggregations @@ -1444,7 +1503,7 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) { buffer_json_member_add_double(wb, "min", r->view.min); buffer_json_member_add_double(wb, "max", r->view.max); - rrdr_timings_v12(wb, "timings", r); + buffer_json_query_timings(wb, "timings", &r->internal.qt->timings); buffer_json_finalize(wb); } @@ -1497,6 +1556,7 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) { } buffer_json_object_close(wb); // view - rrdr_timings_v12(wb, "timings", r); + buffer_json_agents_array_v2(wb, &r->internal.qt->timings, 0); + buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings); buffer_json_finalize(wb); } diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 7d727ce1a1..139fa6ec86 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -3,14 +3,6 @@ #include "web/api/web_api_v1.h" #include "database/storage_engine.h" -inline bool query_target_has_percentage_units(struct query_target *qt) { - if(qt->window.options & RRDR_OPTION_PERCENTAGE || - qt->window.time_group_method == RRDR_GROUPING_CV) - return true; - - return false; -} - void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { rrdset2json(st, wb, NULL, NULL, 0); } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index 7dbcf8bf05..def26c754d 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -61,10 +61,6 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, ti void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options); -struct query_target; -bool query_target_has_percentage_units(struct query_target *qt); -#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW) - int rrdset2value_api_v1( RRDSET *st , BUFFER *wb diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c index ce48c343ca..1d07f62f6a 100644 --- a/web/api/formatters/value/value.c +++ b/web/api/formatters/value/value.c @@ -93,7 +93,8 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host, }; ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr)); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); QUERY_VALUE qv; @@ -143,6 +144,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host, } rrdr_free(owa, r); + query_target_release(qt); onewayalloc_destroy(owa); return qv; diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml index bad4b38f28..153692b80d 100644 --- a/web/api/netdata-swagger.yaml +++ b/web/api/netdata-swagger.yaml @@ -249,6 +249,7 @@ paths: enum: - dimension - instance + - percentage-of-instance - label - node - context diff --git a/web/api/queries/average/average.c b/web/api/queries/average/average.c index a0bed39039..f54dcb243d 100644 --- a/web/api/queries/average/average.c +++ b/web/api/queries/average/average.c @@ -2,55 +2,3 @@ #include "average.h" -// ---------------------------------------------------------------------------- -// average - -struct grouping_average { - NETDATA_DOUBLE sum; - size_t count; -}; - -void grouping_create_average(RRDR *r, const char *options __maybe_unused) { - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_average(RRDR *r) { - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data; - g->sum = 0; - g->count = 0; -} - -void grouping_free_average(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data; - g->sum += value; - g->count++; -} - -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_average *g = (struct grouping_average *)r->time_grouping.data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - if(unlikely(r->time_grouping.resampling_group != 1)) - value = g->sum / r->time_grouping.resampling_divisor; - else - value = g->sum / g->count; - } - - g->sum = 0.0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h index b319668860..2d77cc5714 100644 --- a/web/api/queries/average/average.h +++ b/web/api/queries/average/average.h @@ -6,10 +6,57 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_average(RRDR *r, const char *options __maybe_unused); -void grouping_reset_average(RRDR *r); -void grouping_free_average(RRDR *r); -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// ---------------------------------------------------------------------------- +// average + +struct tg_average { + NETDATA_DOUBLE sum; + size_t count; +}; + +static inline void tg_average_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_average)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_average_reset(RRDR *r) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + g->sum = 0; + g->count = 0; +} + +static inline void tg_average_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_average_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + g->sum += value; + g->count++; +} + +static inline NETDATA_DOUBLE tg_average_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + if(unlikely(r->time_grouping.resampling_group != 1)) + value = g->sum / r->time_grouping.resampling_divisor; + else + value = g->sum / g->count; + } + + g->sum = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_AVERAGE_H diff --git a/web/api/queries/countif/countif.c b/web/api/queries/countif/countif.c index 683f1004e9..8a3a1f50b3 100644 --- a/web/api/queries/countif/countif.c +++ b/web/api/queries/countif/countif.c @@ -5,132 +5,3 @@ // ---------------------------------------------------------------------------- // countif -struct grouping_countif { - size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE); - NETDATA_DOUBLE target; - size_t count; - size_t matched; -}; - -static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v == target); -} - -static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v != target); -} - -static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v < target); -} - -static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v <= target); -} - -static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v > target); -} - -static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v >= target); -} - -void grouping_create_countif(RRDR *r, const char *options __maybe_unused) { - struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif)); - r->time_grouping.data = g; - - if(options && *options) { - // skip any leading spaces - while(isspace(*options)) options++; - - // find the comparison function - switch(*options) { - case '!': - options++; - if(*options != '=' && *options != ':') - options--; - g->comparison = countif_notequal; - break; - - case '>': - options++; - if(*options == '=' || *options == ':') { - g->comparison = countif_greaterequal; - } - else { - options--; - g->comparison = countif_greater; - } - break; - - case '<': - options++; - if(*options == '>') { - g->comparison = countif_notequal; - } - else if(*options == '=' || *options == ':') { - g->comparison = countif_lessequal; - } - else { - options--; - g->comparison = countif_less; - } - break; - - default: - case '=': - case ':': - g->comparison = countif_equal; - break; - } - if(*options) options++; - - // skip everything up to the first digit - while(isspace(*options)) options++; - - g->target = str2ndd(options, NULL); - } - else { - g->target = 0.0; - g->comparison = countif_equal; - } -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_countif(RRDR *r) { - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data; - g->matched = 0; - g->count = 0; -} - -void grouping_free_countif(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data; - g->matched += g->comparison(value, g->target); - g->count++; -} - -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_countif *g = (struct grouping_countif *)r->time_grouping.data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count; - } - - g->matched = 0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h index dfe8056589..896b9d873a 100644 --- a/web/api/queries/countif/countif.h +++ b/web/api/queries/countif/countif.h @@ -6,10 +6,143 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_countif(RRDR *r, const char *options __maybe_unused); -void grouping_reset_countif(RRDR *r); -void grouping_free_countif(RRDR *r); -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +enum tg_countif_cmp { + TG_COUNTIF_EQUAL, + TG_COUNTIF_NOTEQUAL, + TG_COUNTIF_LESS, + TG_COUNTIF_LESSEQUAL, + TG_COUNTIF_GREATER, + TG_COUNTIF_GREATEREQUAL, +}; + +struct tg_countif { + enum tg_countif_cmp comparison; + NETDATA_DOUBLE target; + size_t count; + size_t matched; +}; + +static inline void tg_countif_create(RRDR *r, const char *options __maybe_unused) { + struct tg_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_countif)); + r->time_grouping.data = g; + + if(options && *options) { + // skip any leading spaces + while(isspace(*options)) options++; + + // find the comparison function + switch(*options) { + case '!': + options++; + if(*options != '=' && *options != ':') + options--; + g->comparison = TG_COUNTIF_NOTEQUAL; + break; + + case '>': + options++; + if(*options == '=' || *options == ':') { + g->comparison = TG_COUNTIF_GREATEREQUAL; + } + else { + options--; + g->comparison = TG_COUNTIF_GREATER; + } + break; + + case '<': + options++; + if(*options == '>') { + g->comparison = TG_COUNTIF_NOTEQUAL; + } + else if(*options == '=' || *options == ':') { + g->comparison = TG_COUNTIF_LESSEQUAL; + } + else { + options--; + g->comparison = TG_COUNTIF_LESS; + } + break; + + default: + case '=': + case ':': + g->comparison = TG_COUNTIF_EQUAL; + break; + } + if(*options) options++; + + // skip everything up to the first digit + while(isspace(*options)) options++; + + g->target = str2ndd(options, NULL); + } + else { + g->target = 0.0; + g->comparison = TG_COUNTIF_EQUAL; + } +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_countif_reset(RRDR *r) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + g->matched = 0; + g->count = 0; +} + +static inline void tg_countif_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_countif_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + switch(g->comparison) { + case TG_COUNTIF_GREATER: + if(value > g->target) g->matched++; + break; + + case TG_COUNTIF_GREATEREQUAL: + if(value >= g->target) g->matched++; + break; + + case TG_COUNTIF_LESS: + if(value < g->target) g->matched++; + break; + + case TG_COUNTIF_LESSEQUAL: + if(value <= g->target) g->matched++; + break; + + case TG_COUNTIF_EQUAL: + if(value == g->target) g->matched++; + break; + + case TG_COUNTIF_NOTEQUAL: + if(value != g->target) g->matched++; + break; + } + g->count++; +} + +static inline NETDATA_DOUBLE tg_countif_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count; + } + + g->matched = 0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_COUNTIF_H diff --git a/web/api/queries/des/des.c b/web/api/queries/des/des.c index e2b756b50c..d0e234e238 100644 --- a/web/api/queries/des/des.c +++ b/web/api/queries/des/des.c @@ -6,132 +6,3 @@ // ---------------------------------------------------------------------------- // single exponential smoothing - -struct grouping_des { - NETDATA_DOUBLE alpha; - NETDATA_DOUBLE alpha_other; - NETDATA_DOUBLE beta; - NETDATA_DOUBLE beta_other; - - NETDATA_DOUBLE level; - NETDATA_DOUBLE trend; - - size_t count; -}; - -static size_t max_window_size = 15; - -void grouping_init_des(void) { - long long ret = config_get_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size); - if(ret <= 1) { - config_set_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size); - } - else { - max_window_size = (size_t) ret; - } -} - -static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) { - (void)g; - - NETDATA_DOUBLE points; - if(r->view.group == 1) { - // provide a running DES - points = (NETDATA_DOUBLE)r->time_grouping.points_wanted; - } - else { - // provide a SES with flush points - points = (NETDATA_DOUBLE)r->view.group; - } - - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points; -} - -static inline void set_alpha(RRDR *r, struct grouping_des *g) { - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - - g->alpha = 2.0 / (window(r, g) + 1.0); - g->alpha_other = 1.0 - g->alpha; - - //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha); -} - -static inline void set_beta(RRDR *r, struct grouping_des *g) { - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - - g->beta = 2.0 / (window(r, g) + 1.0); - g->beta_other = 1.0 - g->beta; - - //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); -} - -void grouping_create_des(RRDR *r, const char *options __maybe_unused) { - struct grouping_des *g = (struct grouping_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct grouping_des)); - set_alpha(r, g); - set_beta(r, g); - g->level = 0.0; - g->trend = 0.0; - g->count = 0; - r->time_grouping.data = g; -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_des(RRDR *r) { - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data; - g->level = 0.0; - g->trend = 0.0; - g->count = 0; - - // fprintf(stderr, "\nDES: "); - -} - -void grouping_free_des(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data; - - if(likely(g->count > 0)) { - // we have at least a number so far - - if(unlikely(g->count == 1)) { - // the second value we got - g->trend = value - g->trend; - g->level = value; - } - - // for the values, except the first - NETDATA_DOUBLE last_level = g->level; - g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); - g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); - } - else { - // the first value we got - g->level = g->trend = value; - } - - g->count++; - - //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); -} - -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_des *g = (struct grouping_des *)r->time_grouping.data; - - if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - return 0.0; - } - - //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level); - - return g->level; -} diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h index 05fa01b349..3153d497cb 100644 --- a/web/api/queries/des/des.h +++ b/web/api/queries/des/des.h @@ -6,12 +6,133 @@ #include "../query.h" #include "../rrdr.h" -void grouping_init_des(void); +struct tg_des { + NETDATA_DOUBLE alpha; + NETDATA_DOUBLE alpha_other; + NETDATA_DOUBLE beta; + NETDATA_DOUBLE beta_other; -void grouping_create_des(RRDR *r, const char *options __maybe_unused); -void grouping_reset_des(RRDR *r); -void grouping_free_des(RRDR *r); -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + NETDATA_DOUBLE level; + NETDATA_DOUBLE trend; + + size_t count; +}; + +static size_t tg_des_max_window_size = 15; + +static inline void tg_des_init(void) { + long long ret = config_get_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size); + if(ret <= 1) { + config_set_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size); + } + else { + tg_des_max_window_size = (size_t) ret; + } +} + +static inline NETDATA_DOUBLE tg_des_window(RRDR *r, struct tg_des *g) { + (void)g; + + NETDATA_DOUBLE points; + if(r->view.group == 1) { + // provide a running DES + points = (NETDATA_DOUBLE)r->time_grouping.points_wanted; + } + else { + // provide a SES with flush points + points = (NETDATA_DOUBLE)r->view.group; + } + + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + return (points > (NETDATA_DOUBLE)tg_des_max_window_size) ? (NETDATA_DOUBLE)tg_des_max_window_size : points; +} + +static inline void tg_des_set_alpha(RRDR *r, struct tg_des *g) { + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + + g->alpha = 2.0 / (tg_des_window(r, g) + 1.0); + g->alpha_other = 1.0 - g->alpha; + + //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha); +} + +static inline void tg_des_set_beta(RRDR *r, struct tg_des *g) { + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + + g->beta = 2.0 / (tg_des_window(r, g) + 1.0); + g->beta_other = 1.0 - g->beta; + + //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); +} + +static inline void tg_des_create(RRDR *r, const char *options __maybe_unused) { + struct tg_des *g = (struct tg_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct tg_des)); + tg_des_set_alpha(r, g); + tg_des_set_beta(r, g); + g->level = 0.0; + g->trend = 0.0; + g->count = 0; + r->time_grouping.data = g; +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_des_reset(RRDR *r) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + g->level = 0.0; + g->trend = 0.0; + g->count = 0; + + // fprintf(stderr, "\nDES: "); + +} + +static inline void tg_des_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_des_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + + if(likely(g->count > 0)) { + // we have at least a number so far + + if(unlikely(g->count == 1)) { + // the second value we got + g->trend = value - g->trend; + g->level = value; + } + + // for the values, except the first + NETDATA_DOUBLE last_level = g->level; + g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); + g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); + } + else { + // the first value we got + g->level = g->trend = value; + } + + g->count++; + + //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); +} + +static inline NETDATA_DOUBLE tg_des_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + return 0.0; + } + + //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level); + + return g->level; +} #endif //NETDATA_API_QUERIES_DES_H diff --git a/web/api/queries/incremental_sum/incremental_sum.c b/web/api/queries/incremental_sum/incremental_sum.c index 896f1901ba..88072f2970 100644 --- a/web/api/queries/incremental_sum/incremental_sum.c +++ b/web/api/queries/incremental_sum/incremental_sum.c @@ -5,62 +5,3 @@ // ---------------------------------------------------------------------------- // incremental sum -struct grouping_incremental_sum { - NETDATA_DOUBLE first; - NETDATA_DOUBLE last; - size_t count; -}; - -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) { - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_incremental_sum(RRDR *r) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data; - g->first = 0; - g->last = 0; - g->count = 0; -} - -void grouping_free_incremental_sum(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data; - - if(unlikely(!g->count)) { - g->first = value; - g->count++; - } - else { - g->last = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->time_grouping.data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(unlikely(g->count == 1)) { - value = 0.0; - } - else { - value = g->last - g->first; - } - - g->first = 0.0; - g->last = 0.0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index c24507fcf2..dd6483b2ca 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -6,10 +6,64 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); -void grouping_reset_incremental_sum(RRDR *r); -void grouping_free_incremental_sum(RRDR *r); -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_incremental_sum { + NETDATA_DOUBLE first; + NETDATA_DOUBLE last; + size_t count; +}; + +static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_incremental_sum)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_incremental_sum_reset(RRDR *r) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + g->first = 0; + g->last = 0; + g->count = 0; +} + +static inline void tg_incremental_sum_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_incremental_sum_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + + if(unlikely(!g->count)) { + g->first = value; + g->count++; + } + else { + g->last = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_incremental_sum_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(unlikely(g->count == 1)) { + value = 0.0; + } + else { + value = g->last - g->first; + } + + g->first = 0.0; + g->last = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H diff --git a/web/api/queries/max/max.c b/web/api/queries/max/max.c index b75d054591..cc5999a29b 100644 --- a/web/api/queries/max/max.c +++ b/web/api/queries/max/max.c @@ -5,53 +5,3 @@ // ---------------------------------------------------------------------------- // max -struct grouping_max { - NETDATA_DOUBLE max; - size_t count; -}; - -void grouping_create_max(RRDR *r, const char *options __maybe_unused) { - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_max(RRDR *r) { - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data; - g->max = 0; - g->count = 0; -} - -void grouping_free_max(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data; - - if(!g->count || fabsndd(value) > fabsndd(g->max)) { - g->max = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_max *g = (struct grouping_max *)r->time_grouping.data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = g->max; - } - - g->max = 0.0; - g->count = 0; - - return value; -} - diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h index e2427d26d4..c26bb79ad5 100644 --- a/web/api/queries/max/max.h +++ b/web/api/queries/max/max.h @@ -6,10 +6,54 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_max(RRDR *r, const char *options __maybe_unused); -void grouping_reset_max(RRDR *r); -void grouping_free_max(RRDR *r); -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_max { + NETDATA_DOUBLE max; + size_t count; +}; + +static inline void tg_max_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_max)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_max_reset(RRDR *r) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + g->max = 0; + g->count = 0; +} + +static inline void tg_max_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_max_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + + if(!g->count || fabsndd(value) > fabsndd(g->max)) { + g->max = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_max_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = g->max; + } + + g->max = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_MAX_H diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c index 284253980d..9865b485cd 100644 --- a/web/api/queries/median/median.c +++ b/web/api/queries/median/median.c @@ -4,137 +4,3 @@ // ---------------------------------------------------------------------------- // median - -struct grouping_median { - size_t series_size; - size_t next_pos; - NETDATA_DOUBLE percent; - - NETDATA_DOUBLE *series; -}; - -void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { - long entries = r->view.group; - if(entries < 10) entries = 10; - - struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median)); - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); - g->series_size = (size_t)entries; - - g->percent = def; - if(options && *options) { - g->percent = str2ndd(options, NULL); - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; - if(g->percent < 0.0) g->percent = 0.0; - if(g->percent > 50.0) g->percent = 50.0; - } - - g->percent = g->percent / 100.0; - r->time_grouping.data = g; -} - -void grouping_create_median(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 0.0); -} -void grouping_create_trimmed_median1(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 1.0); -} -void grouping_create_trimmed_median2(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 2.0); -} -void grouping_create_trimmed_median3(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 3.0); -} -void grouping_create_trimmed_median5(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 5.0); -} -void grouping_create_trimmed_median10(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 10.0); -} -void grouping_create_trimmed_median15(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 15.0); -} -void grouping_create_trimmed_median20(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 20.0); -} -void grouping_create_trimmed_median25(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 25.0); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_median(RRDR *r) { - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data; - g->next_pos = 0; -} - -void grouping_free_median(RRDR *r) { - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data; - if(g) onewayalloc_freez(r->internal.owa, g->series); - - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data; - - if(unlikely(g->next_pos >= g->series_size)) { - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); - g->series_size *= 2; - } - - g->series[g->next_pos++] = value; -} - -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_median *g = (struct grouping_median *)r->time_grouping.data; - - size_t available_slots = g->next_pos; - NETDATA_DOUBLE value; - - if(unlikely(!available_slots)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(available_slots == 1) { - value = g->series[0]; - } - else { - sort_series(g->series, available_slots); - - size_t start_slot = 0; - size_t end_slot = available_slots - 1; - - if(g->percent > 0.0) { - NETDATA_DOUBLE min = g->series[0]; - NETDATA_DOUBLE max = g->series[available_slots - 1]; - NETDATA_DOUBLE delta = (max - min) * g->percent; - - NETDATA_DOUBLE wanted_min = min + delta; - NETDATA_DOUBLE wanted_max = max - delta; - - for (start_slot = 0; start_slot < available_slots; start_slot++) - if (g->series[start_slot] >= wanted_min) break; - - for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) - if (g->series[end_slot] <= wanted_max) break; - } - - if(start_slot == end_slot) - value = g->series[start_slot]; - else - value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); - } - - if(unlikely(!netdata_double_isnumber(value))) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - //log_series_to_stderr(g->series, g->next_pos, value, "median"); - - g->next_pos = 0; - - return value; -} diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index 9fc159db42..3d6d359250 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,18 +6,138 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_median(RRDR *r, const char *options); -void grouping_create_trimmed_median1(RRDR *r, const char *options); -void grouping_create_trimmed_median2(RRDR *r, const char *options); -void grouping_create_trimmed_median3(RRDR *r, const char *options); -void grouping_create_trimmed_median5(RRDR *r, const char *options); -void grouping_create_trimmed_median10(RRDR *r, const char *options); -void grouping_create_trimmed_median15(RRDR *r, const char *options); -void grouping_create_trimmed_median20(RRDR *r, const char *options); -void grouping_create_trimmed_median25(RRDR *r, const char *options); -void grouping_reset_median(RRDR *r); -void grouping_free_median(RRDR *r); -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_median { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static inline void tg_median_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->view.group; + if(entries < 10) entries = 10; + + struct tg_median *g = (struct tg_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_median)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = g->percent / 100.0; + r->time_grouping.data = g; +} + +static inline void tg_median_create(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 0.0); +} +static inline void tg_median_create_trimmed_1(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 1.0); +} +static inline void tg_median_create_trimmed_2(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 2.0); +} +static inline void tg_median_create_trimmed_3(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 3.0); +} +static inline void tg_median_create_trimmed_5(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 5.0); +} +static inline void tg_median_create_trimmed_10(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 10.0); +} +static inline void tg_median_create_trimmed_15(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 15.0); +} +static inline void tg_median_create_trimmed_20(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 20.0); +} +static inline void tg_median_create_trimmed_25(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 25.0); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_median_reset(RRDR *r) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + g->next_pos = 0; +} + +static inline void tg_median_free(RRDR *r) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_median_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +static inline NETDATA_DOUBLE tg_median_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + + size_t available_slots = g->next_pos; + NETDATA_DOUBLE value; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + size_t start_slot = 0; + size_t end_slot = available_slots - 1; + + if(g->percent > 0.0) { + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + NETDATA_DOUBLE delta = (max - min) * g->percent; + + NETDATA_DOUBLE wanted_min = min + delta; + NETDATA_DOUBLE wanted_max = max - delta; + + for (start_slot = 0; start_slot < available_slots; start_slot++) + if (g->series[start_slot] >= wanted_min) break; + + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) + if (g->series[end_slot] <= wanted_max) break; + } + + if(start_slot == end_slot) + value = g->series[start_slot]; + else + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "median"); + + g->next_pos = 0; + + return value; +} #endif //NETDATA_API_QUERIES_MEDIAN_H diff --git a/web/api/queries/min/min.c b/web/api/queries/min/min.c index c53aa6e311..cefa7cf317 100644 --- a/web/api/queries/min/min.c +++ b/web/api/queries/min/min.c @@ -5,53 +5,3 @@ // ---------------------------------------------------------------------------- // min -struct grouping_min { - NETDATA_DOUBLE min; - size_t count; -}; - -void grouping_create_min(RRDR *r, const char *options __maybe_unused) { - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_min(RRDR *r) { - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data; - g->min = 0; - g->count = 0; -} - -void grouping_free_min(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->time_grouping.data); - r->time_grouping.data = NULL; -} - -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data; - - if(!g->count || fabsndd(value) < fabsndd(g->min)) { - g->min = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_min *g = (struct grouping_min *)r->time_grouping.data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = g->min; - } - - g->min = 0.0; - g->count = 0; - - return value; -} - diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h index dcdfe252fc..3c53dfd1d7 100644 --- a/web/api/queries/min/min.h +++ b/web/api/queries/min/min.h @@ -6,10 +6,54 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_min(RRDR *r, const char *options __maybe_unused); -void grouping_reset_min(RRDR *r); -void grouping_free_min(RRDR *r); -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_min { + NETDATA_DOUBLE min; + size_t count; +}; + +static inline void tg_min_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_min)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_min_reset(RRDR *r) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + g->min = 0; + g->count = 0; +} + +static inline void tg_min_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_min_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + + if(!g->count || fabsndd(value) < fabsndd(g->min)) { + g->min = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_min_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = g->min; + } + + g->min = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_MIN_H diff --git a/web/api/queries/percentile/percentile.c b/web/api/queries/percentile/percentile.c index b399efdfda..da3b326969 100644 --- a/web/api/queries/percentile/percentile.c +++ b/web/api/queries/percentile/percentile.c @@ -4,166 +4,3 @@ // ---------------------------------------------------------------------------- // median - -struct grouping_percentile { - size_t series_size; - size_t next_pos; - NETDATA_DOUBLE percent; - - NETDATA_DOUBLE *series; -}; - -static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { - long entries = r->view.group; - if(entries < 10) entries = 10; - - struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile)); - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); - g->series_size = (size_t)entries; - - g->percent = def; - if(options && *options) { - g->percent = str2ndd(options, NULL); - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; - if(g->percent < 0.0) g->percent = 0.0; - if(g->percent > 100.0) g->percent = 100.0; - } - - g->percent = g->percent / 100.0; - r->time_grouping.data = g; -} - -void grouping_create_percentile25(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 25.0); -} -void grouping_create_percentile50(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 50.0); -} -void grouping_create_percentile75(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 75.0); -} -void grouping_create_percentile80(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 80.0); -} -void grouping_create_percentile90(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 90.0); -} -void grouping_create_percentile95(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 95.0); -} -void grouping_create_percentile97(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 97.0); -} -void grouping_create_percentile98(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 98.0); -} -void grouping_create_percentile99(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 99.0); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_percentile(RRDR *r) { - struct grouping_percentile *g = (struct grouping_percentile *)r->time_grouping.data; - g->next_pos = 0; -} - -void g