Query Engine multi-granularity support (and MC improvements) (#13155)

* set grouping functions

* storage engine should check the validity of timestamps, not the query engine

* calculate and store in RRDR anomaly rates for every query

* anomaly rate used by volume metric correlations

* mc volume should use absolute data, to avoid cancelling effect

* return anomaly-rates in jasonwrap with jw-anomaly-rates option to data queries

* dont return null on anomaly rates

* allow passing group query options from the URL

* added countif to the query engine and used it in metric correlations

* fix configure

* fix countif and anomaly rate percentages

* added group_options to metric correlations; updated swagger

* added newline at the end of yaml file

* always check the time the highlighted window was above/below the highlighted window

* properly track time in memory queries

* error for internal checks only

* moved pack_storage_number() into the storage engines

* moved unpack_storage_number() inside the storage engines

* remove old comment

* pass unit tests

* properly detect zero or subnormal values in pack_storage_number()

* fill nulls before the value, not after

* make sure math.h is included

* workaround for isfinite()

* fix for isfinite()

* faster isfinite() alternative

* fix for faster isfinite() alternative

* next_metric() now returns end_time too

* variable step implemented in a generic way

* remove left-over variables

* ensure we always complete the wanted number of points

* fixes

* ensure no infinite loop

* mc-volume-improvements: Add information about invalid condition

* points should have a duration in the past

* removed unneeded info() line

* Fix unit tests for exporting engine

* new_point should only be checked when it is fetched from the db; better comment about the premature breaking of the main query loop

Co-authored-by: Thiago Marques <thiagoftsm@gmail.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

4 files changed, 15 insertions, 12 deletions

diff --git a/exporting/process_data.c b/exporting/process_data.c
index 548a7c8bf7..83f0e012ea 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -81,7 +81,6 @@ calculated_number exporting_calculate_value_from_stored_data(
     time_t last_t = rd->state->query_ops.latest_time(rd);
     time_t update_every = st->update_every;
     struct rrddim_query_handle handle;
-    storage_number n;
 
     // step back a little, to make sure we have complete data collection
     // for all metrics
@@ -125,17 +124,18 @@ calculated_number exporting_calculate_value_from_stored_data(
 
     size_t counter = 0;
     calculated_number sum = 0;
+    calculated_number value;
 
     for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
-        time_t curr_t;
-        n = rd->state->query_ops.next_metric(&handle, &curr_t);
+        time_t curr_t, end_t;
+        SN_FLAGS flags;
+        value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
 
-        if (unlikely(!does_storage_number_exist(n))) {
+        if (unlikely(!calculated_number_isnumber(value))) {
             // not collected
             continue;
         }
 
-        calculated_number value = unpack_storage_number(n);
         sum += value;
 
         counter++;
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index 8fba62e783..8838d6574a 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -230,13 +230,16 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
     return mock_type(int);
 }
 
-storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time)
+calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
 {
     (void)handle;
-    (void)current_time;
+    (void)start_time;
+    (void)end_time;
+    (void) flags;
+
 
     function_called();
-    return mock_type(storage_number);
+    return mock_type(calculated_number);
 }
 
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index d0b846429d..e265ef7de9 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -307,19 +307,19 @@ static void test_exporting_calculate_value_from_stored_data(void **state)
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 0);
     expect_function_call(__mock_rrddim_query_next_metric);
-    will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS));
+    will_return(__mock_rrddim_query_next_metric, 27);
 
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 0);
     expect_function_call(__mock_rrddim_query_next_metric);
-    will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS));
+    will_return(__mock_rrddim_query_next_metric, 45);
 
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 1);
 
     expect_function_call(__mock_rrddim_query_finalize);
 
-    assert_int_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36);
+    assert_float_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36, 0.1);
 }
 
 static void test_prepare_buffers(void **state)
diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h
index f8003b0e47..7431e109ba 100644
--- a/exporting/tests/test_exporting_engine.h
+++ b/exporting/tests/test_exporting_engine.h
@@ -61,7 +61,7 @@ time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
 time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
 void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
 int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
-storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time);
+calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
 
 // -----------------------------------------------------------------------