Query Engine multi-granularity support (and MC improvements) (#13155)

* set grouping functions

* storage engine should check the validity of timestamps, not the query engine

* calculate and store in RRDR anomaly rates for every query

* anomaly rate used by volume metric correlations

* mc volume should use absolute data, to avoid cancelling effect

* return anomaly-rates in jasonwrap with jw-anomaly-rates option to data queries

* dont return null on anomaly rates

* allow passing group query options from the URL

* added countif to the query engine and used it in metric correlations

* fix configure

* fix countif and anomaly rate percentages

* added group_options to metric correlations; updated swagger

* added newline at the end of yaml file

* always check the time the highlighted window was above/below the highlighted window

* properly track time in memory queries

* error for internal checks only

* moved pack_storage_number() into the storage engines

* moved unpack_storage_number() inside the storage engines

* remove old comment

* pass unit tests

* properly detect zero or subnormal values in pack_storage_number()

* fill nulls before the value, not after

* make sure math.h is included

* workaround for isfinite()

* fix for isfinite()

* faster isfinite() alternative

* fix for faster isfinite() alternative

* next_metric() now returns end_time too

* variable step implemented in a generic way

* remove left-over variables

* ensure we always complete the wanted number of points

* fixes

* ensure no infinite loop

* mc-volume-improvements: Add information about invalid condition

* points should have a duration in the past

* removed unneeded info() line

* Fix unit tests for exporting engine

* new_point should only be checked when it is fetched from the db; better comment about the premature breaking of the main query loop

Co-authored-by: Thiago Marques <thiagoftsm@gmail.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

2 files changed, 27 insertions, 16 deletions

diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c
index ba7a668745..f5f9cf1ef4 100644
--- a/libnetdata/storage_number/storage_number.c
+++ b/libnetdata/storage_number/storage_number.c
@@ -2,12 +2,7 @@
 
 #include "../libnetdata.h"
 
-#define get_storage_number_flags(value) \
-    ((((storage_number)(value)) & (1 << 24)) | \
-     (((storage_number)(value)) & (1 << 25)) | \
-     (((storage_number)(value)) & (1 << 26)))
-
-storage_number pack_storage_number(calculated_number value, uint32_t flags) {
+storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
     // bit 32 = sign 0:positive, 1:negative
     // bit 31 = 0:divide, 1:multiply
     // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
@@ -16,8 +11,11 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) {
     // bit 25 SN_ANOMALY_BIT = 0: anomalous, 1: not anomalous
     // bit 24 to bit 1 = the value
 
-    storage_number r = get_storage_number_flags(flags);
-    if(!value)
+    storage_number r = flags & SN_ALL_FLAGS;
+
+    // The isnormal() macro shall determine whether its argument value
+    // is normal (neither zero, subnormal, infinite, nor NaN).
+    if(unlikely(!isnormal(value)))
         goto RET_SN;
 
     int m = 0;
diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h
index 09dc42f980..64d59c17c2 100644
--- a/libnetdata/storage_number/storage_number.h
+++ b/libnetdata/storage_number/storage_number.h
@@ -3,6 +3,7 @@
 #ifndef NETDATA_STORAGE_NUMBER_H
 #define NETDATA_STORAGE_NUMBER_H 1
 
+#include <math.h>
 #include "../libnetdata.h"
 
 #ifdef NETDATA_WITHOUT_LONG_DOUBLE
@@ -59,18 +60,29 @@ typedef long double collected_number;
 
 #define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)
 
-#define calculated_number_isnumber(a) (!(fpclassify(a) & (FP_NAN|FP_INFINITE)))
+#if defined(HAVE_ISFINITE) || defined(isfinite)
+// The isfinite() macro shall determine whether its argument has a
+// finite value (zero, subnormal, or normal, and not infinite or NaN).
+#define calculated_number_isnumber(a) (isfinite(a))
+#elif defined(HAVE_FINITE) || defined(finite)
+#define calculated_number_isnumber(a) (finite(a))
+#else
+#define calculated_number_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
+#endif
 
 typedef uint32_t storage_number;
 #define STORAGE_NUMBER_FORMAT "%u"
 
-#define SN_ANOMALY_BIT      (1 << 24) // the anomaly bit of the value
-#define SN_EXISTS_RESET     (1 << 25) // the value has been overflown
-#define SN_EXISTS_100       (1 << 26) // very large value (multiplier is 100 instead of 10)
+typedef enum {
+    SN_ANOMALY_BIT   = (1 << 24), // the anomaly bit of the value
+    SN_EXISTS_RESET  = (1 << 25), // the value has been overflown
+    SN_EXISTS_100    = (1 << 26)  // very large value (multiplier is 100 instead of 10)
+} SN_FLAGS;
 
-#define SN_DEFAULT_FLAGS    SN_ANOMALY_BIT
+#define SN_ALL_FLAGS (SN_ANOMALY_BIT|SN_EXISTS_RESET|SN_EXISTS_100)
 
 #define SN_EMPTY_SLOT 0x00000000
+#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT
 
 // When the calculated number is zero and the value is anomalous (ie. it's bit
 // is zero) we want to return a storage_number representation that is
@@ -83,7 +95,7 @@ typedef uint32_t storage_number;
 #define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)
 #define did_storage_number_reset(value)  ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)
 
-storage_number pack_storage_number(calculated_number value, uint32_t flags);
+storage_number pack_storage_number(calculated_number value, SN_FLAGS flags);
 static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const));
 
 int print_calculated_number(char *str, calculated_number value);
@@ -106,7 +118,8 @@ int print_calculated_number(char *str, calculated_number value);
 static inline calculated_number unpack_storage_number(storage_number value) {
     extern calculated_number unpack_storage_number_lut10x[4 * 8];
 
-    if(!value) return 0;
+    if(unlikely(value == SN_EMPTY_SLOT))
+        return NAN;
 
     int sign = 1, exp = 0;
     int factor = 0;
@@ -127,7 +140,7 @@ static inline calculated_number unpack_storage_number(storage_number value) {
     // bit 25 SN_ANOMALY_BIT
 
     // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
-    int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;
+    int mul = (int)((value & ((1<<29)|(1<<28)|(1<<27))) >> 27);
 
     // bit 24 to bit 1 = the value, so remove all other bits
     value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));