From c270b6067bc91233696dc038c6356c1e384e13cf Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Thu, 3 Nov 2022 11:50:31 +0200 Subject: recalculate last_collected_total (#13945) --- database/rrd.h | 3 --- database/rrddim.c | 5 ++++- database/rrdset.c | 64 ++++++++++++++++++++++++++----------------------------- 3 files changed, 34 insertions(+), 38 deletions(-) (limited to 'database') diff --git a/database/rrd.h b/database/rrd.h index c603649e81..3bdf3515e4 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -607,9 +607,6 @@ struct rrdset { struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) struct timeval last_collected_time; // when did this data set last collected values - total_number collected_total; // used internally to calculate percentages - total_number last_collected_total; // used internally to calculate percentages - size_t rrdlabels_last_saved_version; DICTIONARY *functions_view; // collector functions this rrdset supports, can be NULL diff --git a/database/rrddim.c b/database/rrddim.c index 157c8ebd33..ff6c83aefe 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -601,7 +601,7 @@ struct rrddim_map_save_v019 { long double last_calculated_value; // ignored long double last_stored_value; // ignored long long collected_value; // ignored - long long last_collected_value; // ignored + long long last_collected_value; // load and save long double collected_volume; // ignored long double stored_volume; // ignored void *next; // ignored @@ -624,6 +624,7 @@ void rrddim_memory_file_update(RRDDIM *rd) { rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec; rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec; + rd_on_file->last_collected_value = rd->last_collected_value; } void rrddim_memory_file_free(RRDDIM *rd) { @@ -703,6 +704,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO } if(!reset) { + rd->last_collected_value = rd_on_file->last_collected_value; + if(rd_on_file->algorithm != rd->algorithm) info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.", fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm)); diff --git a/database/rrdset.c b/database/rrdset.c index 749009c076..f17c955adb 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -1510,13 +1510,16 @@ after_first_database_work: if(unlikely(rrdhost_has_rrdpush_sender_enabled(st->rrdhost))) rrdset_done_push(st); + uint32_t has_reset_value = 0; + size_t rda_slots = dictionary_entries(st->rrddim_root_index); struct rda_item *rda_base = rrdset_thread_rda(&rda_slots); size_t dim_id; size_t dimensions = 0; struct rda_item *rda = rda_base; - st->collected_total = 0; + total_number collected_total = 0; + total_number last_collected_total = 0; rrddim_foreach_read(rd, st) { if(rd_dfe.counter >= rda_slots) break; @@ -1535,7 +1538,24 @@ after_first_database_work: // calculate totals if(likely(rd->updated)) { - st->collected_total += rd->collected_value; + // if the new is smaller than the old (an overflow, or reset), set the old equal to the new + // to reset the calculation (it will give zero as the calculation for this second) + if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->last_collected_value > rd->collected_value)) { + debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT + , rrdset_id(st) + , rrddim_name(rd) + , rd->last_collected_value + , rd->collected_value + ); + + if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) + has_reset_value = 1; + + rd->last_collected_value = rd->collected_value; + } + + last_collected_total += rd->last_collected_value; + collected_total += rd->collected_value; if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) { error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st)); @@ -1554,8 +1574,6 @@ after_first_database_work: rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC); rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC); - uint32_t has_reset_value = 0; - // process all dimensions to calculate their values // based on the collected figures only // at this stage we do not interpolate anything @@ -1599,7 +1617,7 @@ after_first_database_work: break; case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: - if(unlikely(!st->collected_total)) + if(unlikely(!collected_total)) rd->calculated_value = 0; else // the percentage of the current value @@ -1607,7 +1625,7 @@ after_first_database_work: rd->calculated_value = (NETDATA_DOUBLE)100 * (NETDATA_DOUBLE)rd->collected_value - / (NETDATA_DOUBLE)st->collected_total; + / (NETDATA_DOUBLE)collected_total; rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100" " * " COLLECTED_NUMBER_FORMAT @@ -1615,7 +1633,7 @@ after_first_database_work: , rrddim_name(rd) , rd->calculated_value , rd->collected_value - , st->collected_total + , collected_total ); break; @@ -1694,31 +1712,15 @@ after_first_database_work: continue; } - // if the new is smaller than the old (an overflow, or reset), set the old equal to the new - // to reset the calculation (it will give zero as the calculation for this second) - if(unlikely(rd->last_collected_value > rd->collected_value)) { - debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT - , rrdset_id(st) - , rrddim_name(rd) - , rd->last_collected_value - , rd->collected_value - ); - - if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS))) - has_reset_value = 1; - - rd->last_collected_value = rd->collected_value; - } - // the percentage of the current increment // over the increment of all dimensions together - if(unlikely(st->collected_total == st->last_collected_total)) + if(unlikely(collected_total == last_collected_total)) rd->calculated_value = 0; else rd->calculated_value = (NETDATA_DOUBLE)100 * (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value) - / (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total); + / (NETDATA_DOUBLE)(collected_total - last_collected_total); rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100" " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" @@ -1726,7 +1728,7 @@ after_first_database_work: , rrddim_name(rd) , rd->calculated_value , rd->collected_value, rd->last_collected_value - , st->collected_total, st->last_collected_total + , collected_total, last_collected_total ); break; @@ -1780,8 +1782,6 @@ after_first_database_work: ); after_second_database_work: - st->last_collected_total = st->collected_total; - for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) { rd = rda->rd; if(unlikely(!rd)) continue; @@ -1953,8 +1953,8 @@ struct rrdset_map_save_v019 { usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load struct timeval last_collected_time; // ignored - long long collected_total; // NEEDS TO BE UPDATED - maintained on load - long long last_collected_total; // NEEDS TO BE UPDATED - maintained on load + long long collected_total; // ignored + long long last_collected_total; // ignored void *rrdfamily; // ignored void *rrdhost; // ignored void *next; // ignored @@ -1978,8 +1978,6 @@ void rrdset_memory_file_update(RRDSET *st) { st_on_file->usec_since_last_update = st->usec_since_last_update; st_on_file->last_updated.tv_sec = st->last_updated.tv_sec; st_on_file->last_updated.tv_usec = st->last_updated.tv_usec; - st_on_file->collected_total = st->collected_total; - st_on_file->last_collected_total = st->last_collected_total; } const char *rrdset_cache_filename(RRDSET *st) { @@ -2071,8 +2069,6 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo st->usec_since_last_update = st_on_file->usec_since_last_update; st->last_updated.tv_sec = st_on_file->last_updated.tv_sec; st->last_updated.tv_usec = st_on_file->last_updated.tv_usec; - st->collected_total = st_on_file->collected_total; - st->last_collected_total = st_on_file->last_collected_total; // link it to st st->st_on_file = st_on_file; -- cgit v1.2.3