diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-04-07 21:25:01 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-04-07 21:25:01 +0300 |
commit | 204dd9ae272445d13f308badb07e99675fa34892 (patch) | |
tree | f42e873c60219b5031dcfc3e076adb2398cdb3fe /web/api/queries/percentile/percentile.h | |
parent | 61925baaf6e2448c641e8e71248a47f7a01c4efd (diff) |
Boost dbengine (#14832)
* configure extent cache size
* workers can now execute up to 10 jobs in a run, boosting query prep and extent reads
* fix dispatched and executing counters
* boost to the max
* increase libuv worker threads
* query prep always get more prio than extent reads; stop processing in batch when dbengine is queue is critical
* fix accounting of query prep
* inlining of time-grouping functions, to speed up queries with billions of points
* make switching based on a local const variable
* print one pending contexts loading message per iteration
* inlined store engine query API
* inlined storage engine data collection api
* inlined all storage engine query ops
* eliminate and inline data collection ops
* simplified query group-by
* more error handling
* optimized partial trimming of group-by queries
* preparative work to support multiple passes of group-by
* more preparative work to support multiple passes of group-by (accepts multiple group-by params)
* unified query timings
* unified query timings - weights endpoint
* query target is no longer a static thread variable - there is a list of cached query targets, each of which of freed every 1000 queries
* fix query memory accounting
* added summary.dimension[].pri and sorted summary.dimensions based on priority and then name
* limit max ACLK WEB response size to 30MB
* the response type should be text/plain
* more preparative work for multiple group-by passes
* create functions for generating group by keys, ids and names
* multiple group-by passes are now supported
* parse group-by options array also with an index
* implemented percentage-of-instance group by function
* family is now merged in multi-node contexts
* prevent uninitialized use
Diffstat (limited to 'web/api/queries/percentile/percentile.h')
-rw-r--r-- | web/api/queries/percentile/percentile.h | 175 |
1 files changed, 162 insertions, 13 deletions
diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h index 65e335c110..0532f9d3f0 100644 --- a/web/api/queries/percentile/percentile.h +++ b/web/api/queries/percentile/percentile.h @@ -6,18 +6,167 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_percentile25(RRDR *r, const char *options); -void grouping_create_percentile50(RRDR *r, const char *options); -void grouping_create_percentile75(RRDR *r, const char *options); -void grouping_create_percentile80(RRDR *r, const char *options); -void grouping_create_percentile90(RRDR *r, const char *options); -void grouping_create_percentile95(RRDR *r, const char *options); -void grouping_create_percentile97(RRDR *r, const char *options); -void grouping_create_percentile98(RRDR *r, const char *options); -void grouping_create_percentile99(RRDR *r, const char *options ); -void grouping_reset_percentile(RRDR *r); -void grouping_free_percentile(RRDR *r); -void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_percentile { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static inline void tg_percentile_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->view.group; + if(entries < 10) entries = 10; + + struct tg_percentile *g = (struct tg_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_percentile)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 100.0) g->percent = 100.0; + } + + g->percent = g->percent / 100.0; + r->time_grouping.data = g; +} + +static inline void tg_percentile_create_25(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 25.0); +} +static inline void tg_percentile_create_50(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 50.0); +} +static inline void tg_percentile_create_75(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 75.0); +} +static inline void tg_percentile_create_80(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 80.0); +} +static inline void tg_percentile_create_90(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 90.0); +} +static inline void tg_percentile_create_95(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 95.0); +} +static inline void tg_percentile_create_97(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 97.0); +} +static inline void tg_percentile_create_98(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 98.0); +} +static inline void tg_percentile_create_99(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 99.0); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_percentile_reset(RRDR *r) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + g->next_pos = 0; +} + +static inline void tg_percentile_free(RRDR *r) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_percentile_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +static inline NETDATA_DOUBLE tg_percentile_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + + NETDATA_DOUBLE value; + size_t available_slots = g->next_pos; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + + if (min != max) { + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); + if(!slots_to_use) slots_to_use = 1; + + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; + + NETDATA_DOUBLE percent_interpolation_slot = 0.0; + NETDATA_DOUBLE percent_last_slot = 0.0; + if(percent_delta > 0.0) { + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; + + percent_interpolation_slot = percent_delta / percent_1slot; + percent_last_slot = 1 - percent_interpolation_slot; + } + + int start_slot, stop_slot, step, last_slot, interpolation_slot; + if(min >= 0.0 && max >= 0.0) { + start_slot = 0; + stop_slot = start_slot + (int)slots_to_use; + last_slot = stop_slot - 1; + interpolation_slot = stop_slot; + step = 1; + } + else { + start_slot = (int)available_slots - 1; + stop_slot = start_slot - (int)slots_to_use; + last_slot = stop_slot + 1; + interpolation_slot = stop_slot; + step = -1; + } + + value = 0.0; + for(int slot = start_slot; slot != stop_slot ; slot += step) + value += g->series[slot]; + + size_t counted = slots_to_use; + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { + value += g->series[interpolation_slot] * percent_interpolation_slot; + value += g->series[last_slot] * percent_last_slot; + counted++; + } + + value = value / (NETDATA_DOUBLE)counted; + } + else + value = min; + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "percentile"); + + g->next_pos = 0; + + return value; +} #endif //NETDATA_API_QUERIES_PERCENTILE_H |