QUERY_TARGET: new query engine for Netdata Agent (#13697)

* initial implementation of QUERY_TARGET

* rrd2rrdr() interface

* rrddim_find_best_tier_for_timeframe() ported

* added dimension filtering

* added db object in query target

* rrd2rrdr() ported

* working on formatters

* working on jsonwrapper

* finally, it compiles...

* 1st run without crashes

* query planer working

* cleanup old code

* review changes

* fix also changing data collection frequency

* fix signess

* fix rrdlabels and dimension ordering

* fixes

* remove unused variable

* ml should accept NULL response from rrd2rrdr()

* number formatting fixes

* more number formatting fixes

* more number formatting fixes

* support mc parallel queries

* formatting and cleanup

* added rrd2rrdr_legacy() as a simplified interface to run a query

* make sure rrdset_find_natural_update_every_for_timeframe() returns a value

* make signed comparisons

* weights endpoint using rrdcontexts

* fix for legacy db modes and cleanup

* fix for chart_ids and remove AR chart from weights endpoint

* Ignore command if not initialized yet

* remove unused members

* properly initialize window

* code cleanup - rrddim linked list is gone; rrdset rwlock is gone too

* reviewed RRDR.internal members

* eliminate unnecessary members of QUERY_TARGET

* more complete query ids; more detailed information on aborted queries

* properly terminate option strings

* query id contains group_options which is controlled by users, so escaping is necessary

* tense in query id

* tense in query id - again

* added the remaining query options to the query id

* Expose hidden option to the dimension

* use the hidden flag when loading context dimensions

* Specify table alias for option

* dont update chart last access time, unless at least a dimension of the chart will be queried

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

3 files changed, 28 insertions, 27 deletions

diff --git a/ml/ADCharts.cc b/ml/ADCharts.cc
index 5a78b97918..5c207ed199 100644
--- a/ml/ADCharts.cc
+++ b/ml/ADCharts.cc
@@ -133,31 +133,32 @@ void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyR
     time_t After = Before - Cfg.AnomalyDetectionQueryDuration;
     RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
 
-    RRDR *R = rrd2rrdr(
-        OWA, HostRateRS,
-        1 /* points wanted */,
-        After,
-        Before,
-        Cfg.AnomalyDetectionGroupingMethod,
-        0 /* resampling time */,
-        Options, "anomaly_rate",
-        NULL /* context param list */,
-        NULL /* group options */,
-        0, /* timeout */
-        0 /* tier */
+    RRDR *R = rrd2rrdr_legacy(
+            OWA, HostRateRS,
+            1 /* points wanted */,
+            After,
+            Before,
+            Cfg.AnomalyDetectionGroupingMethod,
+            0 /* resampling time */,
+            Options, "anomaly_rate",
+            NULL /* group options */,
+            0, /* timeout */
+            0 /* tier */
     );
-    assert(R->d == 1 && R->n == 1 && R->rows == 1);
+    if(R) {
+        assert(R->d == 1 && R->n == 1 && R->rows == 1);
 
-    static thread_local bool PrevAboveThreshold = false;
-    bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
-    bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
-    PrevAboveThreshold = AboveThreshold;
+        static thread_local bool PrevAboveThreshold = false;
+        bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
+        bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
+        PrevAboveThreshold = AboveThreshold;
 
-    rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
-    rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
-    rrdset_done(AnomalyDetectionRS);
+        rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
+        rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
+        rrdset_done(AnomalyDetectionRS);
 
-    rrdr_free(OWA, R);
+        rrdr_free(OWA, R);
+    }
     onewayalloc_destroy(OWA);
 }
 
diff --git a/ml/Query.h b/ml/Query.h
index be388959d5..5910b575b4 100644
--- a/ml/Query.h
+++ b/ml/Query.h
@@ -8,7 +8,7 @@ namespace ml {
 class Query {
 public:
     Query(RRDDIM *RD) : RD(RD), Initialized(false) {
-        Ops = &RD->tiers[0]->query_ops;
+        Ops = RD->tiers[0]->query_ops;
     }
 
     time_t latestTime() {
@@ -42,8 +42,8 @@ private:
     RRDDIM *RD;
     bool Initialized;
 
-    struct rrddim_query_ops *Ops;
-    struct rrddim_query_handle Handle;
+    struct storage_engine_query_ops *Ops;
+    struct storage_engine_query_handle Handle;
 };
 
 } // namespace ml
diff --git a/ml/ml.cc b/ml/ml.cc
index cf2fd59798..d73e91569a 100644
--- a/ml/ml.cc
+++ b/ml/ml.cc
@@ -151,7 +151,7 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
     if (R->rows != 1)
         return;
 
-    if (MaxAnomalyRates < 1 || MaxAnomalyRates >= R->d)
+    if (MaxAnomalyRates < 1 || MaxAnomalyRates >= (int)R->d)
         return;
 
     NETDATA_DOUBLE *CNs = R->v;
@@ -160,12 +160,12 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
     std::vector<std::pair<NETDATA_DOUBLE, int>> V;
 
     V.reserve(R->d);
-    for (int Idx = 0; Idx != R->d; Idx++)
+    for (int Idx = 0; Idx != (int)R->d; Idx++)
         V.emplace_back(CNs[Idx], Idx);
 
     std::sort(V.rbegin(), V.rend());
 
-    for (int Idx = MaxAnomalyRates; Idx != R->d; Idx++) {
+    for (int Idx = MaxAnomalyRates; Idx != (int)R->d; Idx++) {
         int UnsortedIdx = V[Idx].second;
 
         int OldFlags = static_cast<int>(DimFlags[UnsortedIdx]);