Query Engine multi-granularity support (and MC improvements) (#13155)

* set grouping functions

* storage engine should check the validity of timestamps, not the query engine

* calculate and store in RRDR anomaly rates for every query

* anomaly rate used by volume metric correlations

* mc volume should use absolute data, to avoid cancelling effect

* return anomaly-rates in jasonwrap with jw-anomaly-rates option to data queries

* dont return null on anomaly rates

* allow passing group query options from the URL

* added countif to the query engine and used it in metric correlations

* fix configure

* fix countif and anomaly rate percentages

* added group_options to metric correlations; updated swagger

* added newline at the end of yaml file

* always check the time the highlighted window was above/below the highlighted window

* properly track time in memory queries

* error for internal checks only

* moved pack_storage_number() into the storage engines

* moved unpack_storage_number() inside the storage engines

* remove old comment

* pass unit tests

* properly detect zero or subnormal values in pack_storage_number()

* fill nulls before the value, not after

* make sure math.h is included

* workaround for isfinite()

* fix for isfinite()

* faster isfinite() alternative

* fix for faster isfinite() alternative

* next_metric() now returns end_time too

* variable step implemented in a generic way

* remove left-over variables

* ensure we always complete the wanted number of points

* fixes

* ensure no infinite loop

* mc-volume-improvements: Add information about invalid condition

* points should have a duration in the past

* removed unneeded info() line

* Fix unit tests for exporting engine

* new_point should only be checked when it is fetched from the db; better comment about the premature breaking of the main query loop

Co-authored-by: Thiago Marques <thiagoftsm@gmail.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

2 files changed, 8 insertions, 56 deletions

diff --git a/ml/Dimension.cc b/ml/Dimension.cc
index 3146e45a6c..7db8174dcf 100644
--- a/ml/Dimension.cc
+++ b/ml/Dimension.cc
@@ -6,55 +6,6 @@
 
 using namespace ml;
 
-/*
- * Copy of the unpack_storage_number which allows us to convert
- * a storage_number to double.
- */
-static CalculatedNumber unpack_storage_number_dbl(storage_number value) {
-    if(!value)
-        return 0;
-
-    int sign = 0, exp = 0;
-    int factor = 10;
-
-    // bit 32 = 0:positive, 1:negative
-    if(unlikely(value & (1 << 31)))
-        sign = 1;
-
-    // bit 31 = 0:divide, 1:multiply
-    if(unlikely(value & (1 << 30)))
-        exp = 1;
-
-    // bit 27 SN_EXISTS_100
-    if(unlikely(value & (1 << 26)))
-        factor = 100;
-
-    // bit 26 SN_EXISTS_RESET
-    // bit 25 SN_ANOMALY_BIT
-
-    // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
-    int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;
-
-    // bit 24 to bit 1 = the value, so remove all other bits
-    value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
-
-    CalculatedNumber CN = value;
-
-    if(exp) {
-        for(; mul; mul--)
-            CN *= factor;
-    }
-    else {
-        for( ; mul ; mul--)
-            CN /= 10;
-    }
-
-    if(sign)
-        CN = -CN;
-
-    return CN;
-}
-
 std::pair<CalculatedNumber *, size_t>
 TrainableDimension::getCalculatedNumbers() {
     size_t MinN = Cfg.MinTrainSamples;
@@ -89,10 +40,10 @@ TrainableDimension::getCalculatedNumbers() {
             break;
 
         auto P = Q.nextMetric();
-        storage_number SN = P.second;
+        CalculatedNumber Value = P.second;
 
-        if (does_storage_number_exist(SN)) {
-            CNs[Idx] = unpack_storage_number_dbl(SN);
+        if (calculated_number_isnumber(Value)) {
+            CNs[Idx] = Value;
             LastValue = CNs[Idx];
             CollectedValues++;
         } else
diff --git a/ml/Query.h b/ml/Query.h
index 8b84bb73e2..ed339fa868 100644
--- a/ml/Query.h
+++ b/ml/Query.h
@@ -27,10 +27,11 @@ public:
         return Ops->is_finished(&Handle);
     }
 
-    std::pair<time_t, storage_number> nextMetric() {
-        time_t CurrT;
-        storage_number SN = Ops->next_metric(&Handle, &CurrT);
-        return { CurrT, SN };
+    std::pair<time_t, CalculatedNumber> nextMetric() {
+        time_t CurrT, EndT;
+        SN_FLAGS Flags;
+        auto Value = (CalculatedNumber)Ops->next_metric(&Handle, &CurrT, &EndT, &Flags);
+        return { CurrT, Value };
     }
 
     ~Query() {