recalculate last_collected_total (#13945)

3 files changed, 34 insertions, 38 deletions

diff --git a/database/rrd.h b/database/rrd.h
index c603649e81..3bdf3515e4 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -607,9 +607,6 @@ struct rrdset {
     struct timeval last_updated;                    // when this data set was last updated (updated every time the rrd_stats_done() function)
     struct timeval last_collected_time;             // when did this data set last collected values
 
-    total_number collected_total;                   // used internally to calculate percentages
-    total_number last_collected_total;              // used internally to calculate percentages
-
     size_t rrdlabels_last_saved_version;
 
     DICTIONARY *functions_view;                     // collector functions this rrdset supports, can be NULL
diff --git a/database/rrddim.c b/database/rrddim.c
index 157c8ebd33..ff6c83aefe 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -601,7 +601,7 @@ struct rrddim_map_save_v019 {
     long double last_calculated_value;              // ignored
     long double last_stored_value;                  // ignored
     long long collected_value;                      // ignored
-    long long last_collected_value;                 // ignored
+    long long last_collected_value;                 // load and save
     long double collected_volume;                   // ignored
     long double stored_volume;                      // ignored
     void *next;                                     // ignored
@@ -624,6 +624,7 @@ void rrddim_memory_file_update(RRDDIM *rd) {
 
     rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
     rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
+    rd_on_file->last_collected_value = rd->last_collected_value;
 }
 
 void rrddim_memory_file_free(RRDDIM *rd) {
@@ -703,6 +704,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
     }
 
     if(!reset) {
+        rd->last_collected_value = rd_on_file->last_collected_value;
+
         if(rd_on_file->algorithm != rd->algorithm)
             info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
                  fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
diff --git a/database/rrdset.c b/database/rrdset.c
index 749009c076..f17c955adb 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -1510,13 +1510,16 @@ after_first_database_work:
     if(unlikely(rrdhost_has_rrdpush_sender_enabled(st->rrdhost)))
         rrdset_done_push(st);
 
+    uint32_t has_reset_value = 0;
+
     size_t rda_slots = dictionary_entries(st->rrddim_root_index);
     struct rda_item *rda_base = rrdset_thread_rda(&rda_slots);
 
     size_t dim_id;
     size_t dimensions = 0;
     struct rda_item *rda = rda_base;
-    st->collected_total = 0;
+    total_number collected_total = 0;
+    total_number last_collected_total = 0;
     rrddim_foreach_read(rd, st) {
         if(rd_dfe.counter >= rda_slots)
             break;
@@ -1535,7 +1538,24 @@ after_first_database_work:
 
         // calculate totals
         if(likely(rd->updated)) {
-            st->collected_total += rd->collected_value;
+            // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
+            // to reset the calculation (it will give zero as the calculation for this second)
+            if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->last_collected_value > rd->collected_value)) {
+                debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
+                , rrdset_id(st)
+                , rrddim_name(rd)
+                , rd->last_collected_value
+                , rd->collected_value
+                );
+
+                if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
+                    has_reset_value = 1;
+
+                rd->last_collected_value = rd->collected_value;
+            }
+
+            last_collected_total += rd->last_collected_value;
+            collected_total += rd->collected_value;
 
             if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE))) {
                 error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
@@ -1554,8 +1574,6 @@ after_first_database_work:
     rrdset_debug(st, "last_stored_ut  = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
     rrdset_debug(st, "next_store_ut   = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
 
-    uint32_t has_reset_value = 0;
-
     // process all dimensions to calculate their values
     // based on the collected figures only
     // at this stage we do not interpolate anything
@@ -1599,7 +1617,7 @@ after_first_database_work:
                 break;
 
             case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
-                if(unlikely(!st->collected_total))
+                if(unlikely(!collected_total))
                     rd->calculated_value = 0;
                 else
                     // the percentage of the current value
@@ -1607,7 +1625,7 @@ after_first_database_work:
                     rd->calculated_value =
                             (NETDATA_DOUBLE)100
                             * (NETDATA_DOUBLE)rd->collected_value
-                            / (NETDATA_DOUBLE)st->collected_total;
+                            / (NETDATA_DOUBLE)collected_total;
 
                 rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
                             " * " COLLECTED_NUMBER_FORMAT
@@ -1615,7 +1633,7 @@ after_first_database_work:
                           , rrddim_name(rd)
                           , rd->calculated_value
                           , rd->collected_value
-                          , st->collected_total
+                          , collected_total
                 );
                 break;
 
@@ -1694,31 +1712,15 @@ after_first_database_work:
                     continue;
                 }
 
-                // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
-                // to reset the calculation (it will give zero as the calculation for this second)
-                if(unlikely(rd->last_collected_value > rd->collected_value)) {
-                    debug(D_RRD_STATS, "'%s' / '%s': RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
-                          , rrdset_id(st)
-                          , rrddim_name(rd)
-                          , rd->last_collected_value
-                          , rd->collected_value
-                    );
-
-                    if(!(rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)))
-                        has_reset_value = 1;
-
-                    rd->last_collected_value = rd->collected_value;
-                }
-
                 // the percentage of the current increment
                 // over the increment of all dimensions together
-                if(unlikely(st->collected_total == st->last_collected_total))
+                if(unlikely(collected_total == last_collected_total))
                     rd->calculated_value = 0;
                 else
                     rd->calculated_value =
                             (NETDATA_DOUBLE)100
                             * (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value)
-                            / (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total);
+                            / (NETDATA_DOUBLE)(collected_total - last_collected_total);
 
                 rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
                             " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
@@ -1726,7 +1728,7 @@ after_first_database_work:
                           , rrddim_name(rd)
                           , rd->calculated_value
                           , rd->collected_value, rd->last_collected_value
-                          , st->collected_total, st->last_collected_total
+                          , collected_total, last_collected_total
                 );
                 break;
 
@@ -1780,8 +1782,6 @@ after_first_database_work:
     );
 
 after_second_database_work:
-    st->last_collected_total  = st->collected_total;
-
     for(dim_id = 0, rda = rda_base ; dim_id < rda_slots ; ++dim_id, ++rda) {
         rd = rda->rd;
         if(unlikely(!rd)) continue;
@@ -1953,8 +1953,8 @@ struct rrdset_map_save_v019 {
     usec_t usec_since_last_update;                  // NEEDS TO BE UPDATED - maintained on load
     struct timeval last_updated;                    // NEEDS TO BE UPDATED - check to reset all - fixed on load
     struct timeval last_collected_time;             // ignored
-    long long collected_total;                      // NEEDS TO BE UPDATED - maintained on load
-    long long last_collected_total;                 // NEEDS TO BE UPDATED - maintained on load
+    long long collected_total;                      // ignored
+    long long last_collected_total;                 // ignored
     void *rrdfamily;                                // ignored
     void *rrdhost;                                  // ignored
     void *next;                                     // ignored
@@ -1978,8 +1978,6 @@ void rrdset_memory_file_update(RRDSET *st) {
     st_on_file->usec_since_last_update = st->usec_since_last_update;
     st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
     st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
-    st_on_file->collected_total = st->collected_total;
-    st_on_file->last_collected_total = st->last_collected_total;
 }
 
 const char *rrdset_cache_filename(RRDSET *st) {
@@ -2071,8 +2069,6 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo
     st->usec_since_last_update = st_on_file->usec_since_last_update;
     st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
     st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
-    st->collected_total = st_on_file->collected_total;
-    st->last_collected_total = st_on_file->last_collected_total;
 
     // link it to st
     st->st_on_file = st_on_file;