Multi-Tier database backend for long term metrics storage (#13263)

* Tier part 1

* Tier part 2

* Tier part 3

* Tier part 4

* Tier part 5

* Fix some ML compilation errors

* fix more conflicts

* pass proper tier

* move metric_uuid from state to RRDDIM

* move aclk_live_status from state to RRDDIM

* move ml_dimension from state to RRDDIM

* abstracted the data collection interface

* support flushing for mem db too

* abstracted the query api

* abstracted latest/oldest time per metric

* cleanup

* store_metric for tier1

* fix for store_metric

* allow multiple tiers, more than 2

* state to tier

* Change storage type in db. Query param to request min, max, sum or average

* Store tier data correctly

* Fix skipping tier page type

* Add tier grouping in the tier

* Fix to handle archived charts (part 1)

* Temp fix for query granularity when requesting tier1 data

* Fix parameters in the correct order and calculate the anomaly based on the anomaly count

* Proper tiering grouping

* Anomaly calculation based on anomaly count

* force type checking on storage handles

* update cmocka tests

* fully dynamic number of storage tiers

* fix static allocation

* configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode

* use default page dt using the tiering info

* automatic selection of tier

* fix for automatic selection of tier

* working prototype of dynamic tier selection

* automatic selection of tier done right (I hope)

* ask for the proper tier value, based on the grouping function

* fixes for unittests and load_metric_next()

* fixes for lgtm findings

* minor renames

* add dbengine to page cache size setting

* add dbengine to page cache with malloc

* query engine optimized to loop as little are required based on the view_update_every

* query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA

* report db points per tier in jsonwrap

* query planer that switches database tiers on the fly to satisfy the query for the entire timeframe

* dbegnine statistics and documentation (in progress)

* calculate average point duration in db

* handle single point pages the best we can

* handle single point pages even better

* Keep page type in the rrdeng_page_descr

* updated doc

* handle future backwards compatibility - improved statistics

* support &tier=X in queries

* enfore increasing iterations on tiers

* tier 1 is always 1 iteration

* backfilling higher tiers on first data collection

* reversed anomaly bit

* set up to 5 tiers

* natural points should only be offered on tier 0, except a specific tier is selected

* do not allow more than 65535 points of tier0 to be aggregated on any tier

* Work only on actually activated tiers

* fix query interpolation

* fix query interpolation again

* fix lgtm finding

* Activate one tier for now

* backfilling of higher tiers using raw metrics from lower tiers

* fix for crash on start when storage tiers is increased from the default

* more statistics on exit

* fix bug that prevented higher tiers to get any values; added backfilling options

* fixed the statistics log line

* removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle

* fixed division by zero on zero points_wanted

* removed dead code

* Decide on the descr->type for the type of metric

* dont store metrics on unknown page types

* free db_metric_handle on sql based context queries

* Disable STORAGE_POINT value check in the exporting engine unit tests

* fix for db modes other than dbengine

* fix for aclk archived chart queries destroying db_metric_handles of valid rrddims

* fix left-over freez() instead of OWA freez on median queries

Co-authored-by: Costa Tsaousis <costa@netdata.cloud>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

5 files changed, 32 insertions, 38 deletions

diff --git a/exporting/process_data.c b/exporting/process_data.c
index f70b39294c..d9154f81b6 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
     time_t before = instance->before;
 
     // find the edges of the rrd database for this chart
-    time_t first_t = rd->state->query_ops.oldest_time(rd);
-    time_t last_t = rd->state->query_ops.latest_time(rd);
+    time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
+    time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
     time_t update_every = st->update_every;
     struct rrddim_query_handle handle;
 
@@ -124,23 +124,20 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
 
     size_t counter = 0;
     NETDATA_DOUBLE sum = 0;
-    NETDATA_DOUBLE value;
 
-    for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
-        time_t curr_t, end_t;
-        SN_FLAGS flags;
-        value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
+    for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) {
+        STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
 
-        if (unlikely(!netdata_double_isnumber(value))) {
+        if (unlikely(storage_point_is_empty(sp))) {
             // not collected
             continue;
         }
 
-        sum += value;
-
-        counter++;
+        sum += sp.sum;
+        counter += sp.count;
     }
-    rd->state->query_ops.finalize(&handle);
+    rd->tiers[0]->query_ops.finalize(&handle);
+
     if (unlikely(!counter)) {
         debug(
             D_EXPORTING,
diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c
index 4f5bd4b277..aae1c53fb2 100644
--- a/exporting/tests/exporting_fixtures.c
+++ b/exporting/tests/exporting_fixtures.c
@@ -63,13 +63,13 @@ int setup_rrdhost()
     rd->collections_counter++;
     rd->next = NULL;
 
-    rd->state = calloc(1, sizeof(*rd->state));
-    rd->state->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
-    rd->state->query_ops.latest_time = __mock_rrddim_query_latest_time;
-    rd->state->query_ops.init = __mock_rrddim_query_init;
-    rd->state->query_ops.is_finished = __mock_rrddim_query_is_finished;
-    rd->state->query_ops.next_metric = __mock_rrddim_query_next_metric;
-    rd->state->query_ops.finalize = __mock_rrddim_query_finalize;
+    rd->tiers[0] = calloc(1, sizeof(struct rrddim_tier));
+    rd->tiers[0]->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
+    rd->tiers[0]->query_ops.latest_time = __mock_rrddim_query_latest_time;
+    rd->tiers[0]->query_ops.init = __mock_rrddim_query_init;
+    rd->tiers[0]->query_ops.is_finished = __mock_rrddim_query_is_finished;
+    rd->tiers[0]->query_ops.next_metric = __mock_rrddim_query_next_metric;
+    rd->tiers[0]->query_ops.finalize = __mock_rrddim_query_finalize;
 
     return 0;
 }
@@ -79,7 +79,7 @@ int teardown_rrdhost()
     RRDDIM *rd = localhost->rrdset_root->dimensions;
     free((void *)rd->name);
     free((void *)rd->id);
-    free(rd->state);
+    free(rd->tiers[0]);
     free(rd);
 
     RRDSET *st = localhost->rrdset_root;
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index 066f52f215..ee36e887a2 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -196,26 +196,27 @@ void rrdset_update_heterogeneous_flag(RRDSET *st)
     (void)st;
 }
 
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd)
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
 {
-    (void)rd;
+    (void)db_metric_handle;
 
     function_called();
     return mock_type(time_t);
 }
 
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd)
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
 {
-    (void)rd;
+    (void)db_metric_handle;
 
     function_called();
     return mock_type(time_t);
 }
 
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time)
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
 {
-    (void)rd;
+    (void)db_metric_handle;
     (void)handle;
+    (void)tier_query_fetch_type;
 
     function_called();
     check_expected(start_time);
@@ -230,16 +231,14 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
     return mock_type(int);
 }
 
-NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle)
 {
     (void)handle;
-    (void)start_time;
-    (void)end_time;
-    (void) flags;
-
 
     function_called();
-    return mock_type(NETDATA_DOUBLE);
+
+    STORAGE_POINT sp = {};
+    return sp;    
 }
 
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index e265ef7de9..bc00e34e1a 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -307,12 +307,10 @@ static void test_exporting_calculate_value_from_stored_data(void **state)
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 0);
     expect_function_call(__mock_rrddim_query_next_metric);
-    will_return(__mock_rrddim_query_next_metric, 27);
 
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 0);
     expect_function_call(__mock_rrddim_query_next_metric);
-    will_return(__mock_rrddim_query_next_metric, 45);
 
     expect_function_call(__mock_rrddim_query_is_finished);
     will_return(__mock_rrddim_query_is_finished, 1);
diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h
index 298a77a9fd..2072dc64f8 100644
--- a/exporting/tests/test_exporting_engine.h
+++ b/exporting/tests/test_exporting_engine.h
@@ -57,11 +57,11 @@ int __wrap_connect_to_one_of(
 void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
 void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line);
 void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line);
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
 int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
-NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle);
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
 
 // -----------------------------------------------------------------------