diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-06-22 11:19:08 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-06-22 11:19:08 +0300 |
commit | b32ca44319e35eb38e5858730f2ea44ea2268926 (patch) | |
tree | 40814c014d919657f6c96d55c0947631a1fb86ee /database/rrdset.c | |
parent | 7b6e23e98c7c5624a76f8dd3435b41594fb5f39f (diff) |
Query Engine multi-granularity support (and MC improvements) (#13155)
* set grouping functions
* storage engine should check the validity of timestamps, not the query engine
* calculate and store in RRDR anomaly rates for every query
* anomaly rate used by volume metric correlations
* mc volume should use absolute data, to avoid cancelling effect
* return anomaly-rates in jasonwrap with jw-anomaly-rates option to data queries
* dont return null on anomaly rates
* allow passing group query options from the URL
* added countif to the query engine and used it in metric correlations
* fix configure
* fix countif and anomaly rate percentages
* added group_options to metric correlations; updated swagger
* added newline at the end of yaml file
* always check the time the highlighted window was above/below the highlighted window
* properly track time in memory queries
* error for internal checks only
* moved pack_storage_number() into the storage engines
* moved unpack_storage_number() inside the storage engines
* remove old comment
* pass unit tests
* properly detect zero or subnormal values in pack_storage_number()
* fill nulls before the value, not after
* make sure math.h is included
* workaround for isfinite()
* fix for isfinite()
* faster isfinite() alternative
* fix for faster isfinite() alternative
* next_metric() now returns end_time too
* variable step implemented in a generic way
* remove left-over variables
* ensure we always complete the wanted number of points
* fixes
* ensure no infinite loop
* mc-volume-improvements: Add information about invalid condition
* points should have a duration in the past
* removed unneeded info() line
* Fix unit tests for exporting engine
* new_point should only be checked when it is fetched from the db; better comment about the premature breaking of the main query loop
Co-authored-by: Thiago Marques <thiagoftsm@gmail.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>
Diffstat (limited to 'database/rrdset.c')
-rw-r--r-- | database/rrdset.c | 51 |
1 files changed, 5 insertions, 46 deletions
diff --git a/database/rrdset.c b/database/rrdset.c index d872f66e2e..fe4ccc68bd 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -1130,7 +1130,7 @@ static inline size_t rrdset_done_interpolate( size_t counter = st->counter; long current_entry = st->current_entry; - uint32_t storage_flags = SN_DEFAULT_FLAGS; + SN_FLAGS storage_flags = SN_DEFAULT_FLAGS; if (has_reset_value) storage_flags |= SN_EXISTS_RESET; @@ -1234,8 +1234,7 @@ static inline size_t rrdset_done_interpolate( if(unlikely(!store_this_entry)) { (void) ml_is_anomalous(rd, 0, false); - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); -// rd->values[current_entry] = SN_EMPTY_SLOT; + rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT); continue; } @@ -1247,18 +1246,8 @@ static inline size_t rrdset_done_interpolate( dim_storage_flags &= ~ ((uint32_t) SN_ANOMALY_BIT); } - rd->state->collect_ops.store_metric(rd, next_store_ut, pack_storage_number(new_value, dim_storage_flags)); -// rd->values[current_entry] = pack_storage_number(new_value, storage_flags ); + rd->state->collect_ops.store_metric(rd, next_store_ut, new_value, dim_storage_flags); rd->last_stored_value = new_value; - - #ifdef NETDATA_INTERNAL_CHECKS - rrdset_debug(st, "%s: STORE[%ld] " - CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT - , rd->name - , current_entry - , unpack_storage_number(rd->values[current_entry]), new_value - ); - #endif } else { (void) ml_is_anomalous(rd, 0, false); @@ -1270,42 +1259,11 @@ static inline size_t rrdset_done_interpolate( ); #endif -// rd->values[current_entry] = SN_EMPTY_SLOT; - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); + rd->state->collect_ops.store_metric(rd, next_store_ut, NAN, SN_EMPTY_SLOT); rd->last_stored_value = NAN; } stored_entries++; - - #ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) { - calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; - calculated_number t2 = unpack_storage_number(rd->values[current_entry]); - - calculated_number accuracy = accuracy_loss(t1, t2); - debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" - , st->id, rd->name - , current_entry - , t2 - , t1 - , accuracy - , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : "" - ); - - rd->collected_volume += t1; - rd->stored_volume += t2; - - accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); - debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" - , st->id, rd->name - , current_entry - , rd->stored_volume - , rd->collected_volume - , accuracy - , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : "" - ); - } - #endif } // reset the storage flags for the next point, if any; storage_flags = SN_DEFAULT_FLAGS; @@ -1503,6 +1461,7 @@ void rrdset_done(RRDSET *st) { // and we have collected metrics for this chart in the past (st->counter != 0) // fill the gap (the chart has been just loaded from disk) if(unlikely(st->counter) && st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { + // TODO this should be inside the storage engine rrdset_done_fill_the_gap(st); last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC; |