Multi-Tier database backend for long term metrics storage (#13263)

* Tier part 1

* Tier part 2

* Tier part 3

* Tier part 4

* Tier part 5

* Fix some ML compilation errors

* fix more conflicts

* pass proper tier

* move metric_uuid from state to RRDDIM

* move aclk_live_status from state to RRDDIM

* move ml_dimension from state to RRDDIM

* abstracted the data collection interface

* support flushing for mem db too

* abstracted the query api

* abstracted latest/oldest time per metric

* cleanup

* store_metric for tier1

* fix for store_metric

* allow multiple tiers, more than 2

* state to tier

* Change storage type in db. Query param to request min, max, sum or average

* Store tier data correctly

* Fix skipping tier page type

* Add tier grouping in the tier

* Fix to handle archived charts (part 1)

* Temp fix for query granularity when requesting tier1 data

* Fix parameters in the correct order and calculate the anomaly based on the anomaly count

* Proper tiering grouping

* Anomaly calculation based on anomaly count

* force type checking on storage handles

* update cmocka tests

* fully dynamic number of storage tiers

* fix static allocation

* configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode

* use default page dt using the tiering info

* automatic selection of tier

* fix for automatic selection of tier

* working prototype of dynamic tier selection

* automatic selection of tier done right (I hope)

* ask for the proper tier value, based on the grouping function

* fixes for unittests and load_metric_next()

* fixes for lgtm findings

* minor renames

* add dbengine to page cache size setting

* add dbengine to page cache with malloc

* query engine optimized to loop as little are required based on the view_update_every

* query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA

* report db points per tier in jsonwrap

* query planer that switches database tiers on the fly to satisfy the query for the entire timeframe

* dbegnine statistics and documentation (in progress)

* calculate average point duration in db

* handle single point pages the best we can

* handle single point pages even better

* Keep page type in the rrdeng_page_descr

* updated doc

* handle future backwards compatibility - improved statistics

* support &tier=X in queries

* enfore increasing iterations on tiers

* tier 1 is always 1 iteration

* backfilling higher tiers on first data collection

* reversed anomaly bit

* set up to 5 tiers

* natural points should only be offered on tier 0, except a specific tier is selected

* do not allow more than 65535 points of tier0 to be aggregated on any tier

* Work only on actually activated tiers

* fix query interpolation

* fix query interpolation again

* fix lgtm finding

* Activate one tier for now

* backfilling of higher tiers using raw metrics from lower tiers

* fix for crash on start when storage tiers is increased from the default

* more statistics on exit

* fix bug that prevented higher tiers to get any values; added backfilling options

* fixed the statistics log line

* removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle

* fixed division by zero on zero points_wanted

* removed dead code

* Decide on the descr->type for the type of metric

* dont store metrics on unknown page types

* free db_metric_handle on sql based context queries

* Disable STORAGE_POINT value check in the exporting engine unit tests

* fix for db modes other than dbengine

* fix for aclk archived chart queries destroying db_metric_handles of valid rrddims

* fix left-over freez() instead of OWA freez on median queries

Co-authored-by: Costa Tsaousis <costa@netdata.cloud>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

3 files changed, 14 insertions, 16 deletions

diff --git a/ml/Dimension.h b/ml/Dimension.h
index e4f8bd1c70..4fbc09b981 100644
--- a/ml/Dimension.h
+++ b/ml/Dimension.h
@@ -12,13 +12,13 @@ namespace ml {
 
 class RrdDimension {
 public:
-    RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->state->query_ops) { }
+    RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->tiers[0]->query_ops) { }
 
     RRDDIM *getRD() const { return RD; }
 
-    time_t latestTime() { return Ops->latest_time(RD); }
+    time_t latestTime() { return Ops->latest_time(RD->tiers[0]->db_metric_handle); }
 
-    time_t oldestTime() { return Ops->oldest_time(RD); }
+    time_t oldestTime() { return Ops->oldest_time(RD->tiers[0]->db_metric_handle); }
 
     unsigned updateEvery() const { return RD->update_every; }
 
diff --git a/ml/Query.h b/ml/Query.h
index ed339fa868..24c5fa3841 100644
--- a/ml/Query.h
+++ b/ml/Query.h
@@ -8,19 +8,19 @@ namespace ml {
 class Query {
 public:
     Query(RRDDIM *RD) : RD(RD) {
-        Ops = &RD->state->query_ops;
+        Ops = &RD->tiers[0]->query_ops;
     }
 
     time_t latestTime() {
-        return Ops->latest_time(RD);
+        return Ops->latest_time(RD->tiers[0]->db_metric_handle);
     }
 
     time_t oldestTime() {
-        return Ops->oldest_time(RD);
+        return Ops->oldest_time(RD->tiers[0]->db_metric_handle);
     }
 
     void init(time_t AfterT, time_t BeforeT) {
-        Ops->init(RD, &Handle, AfterT, BeforeT);
+        Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM);
     }
 
     bool isFinished() {
@@ -28,10 +28,8 @@ public:
     }
 
     std::pair<time_t, CalculatedNumber> nextMetric() {
-        time_t CurrT, EndT;
-        SN_FLAGS Flags;
-        auto Value = (CalculatedNumber)Ops->next_metric(&Handle, &CurrT, &EndT, &Flags);
-        return { CurrT, Value };
+        STORAGE_POINT sp = Ops->next_metric(&Handle);
+        return { sp.start_time, sp.sum / sp.count };
     }
 
     ~Query() {
diff --git a/ml/ml.cc b/ml/ml.cc
index 81de77ebdc..7275d88b87 100644
--- a/ml/ml.cc
+++ b/ml/ml.cc
@@ -80,12 +80,12 @@ void ml_new_dimension(RRDDIM *RD) {
         return;
 
     Dimension *D = new Dimension(RD);
-    RD->state->ml_dimension = static_cast<ml_dimension_t>(D);
+    RD->ml_dimension = static_cast<ml_dimension_t>(D);
     H->addDimension(D);
 }
 
 void ml_delete_dimension(RRDDIM *RD) {
-    Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
+    Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
     if (!D)
         return;
 
@@ -95,7 +95,7 @@ void ml_delete_dimension(RRDDIM *RD) {
     else
         H->removeDimension(D);
 
-    RD->state->ml_dimension = nullptr;
+    RD->ml_dimension = nullptr;
 }
 
 char *ml_get_host_info(RRDHOST *RH) {
@@ -125,7 +125,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) {
 }
 
 bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
-    Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
+    Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
     if (!D)
         return false;
 
@@ -210,7 +210,7 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
 void ml_dimension_update_name(RRDSET *RS, RRDDIM *RD, const char *Name) {
     (void) RS;
 
-    Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
+    Dimension *D = static_cast<Dimension *>(RD->ml_dimension);
     if (!D)
         return;