netdata doubles (#13217)

* netdata doubles

* fix cmocka test

* fix cmocka test again

* fix left-overs of long double to NETDATA_DOUBLE

* RRDDIM detached from disk representation; db settings in [db] section of netdata.conf

* update the memory before saving

* rrdset is now detached from file structures too

* on memory mode map, update the memory mapped structures on every iteration

* allow RRD_ID_LENGTH_MAX to be changed

* granularity secs, back to update every

* fix formatting

* more formatting

3 files changed, 148 insertions, 85 deletions

diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c
index f5f9cf1ef4..f9e2d6fcd7 100644
--- a/libnetdata/storage_number/storage_number.c
+++ b/libnetdata/storage_number/storage_number.c
@@ -2,7 +2,7 @@
 
 #include "../libnetdata.h"
 
-storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
+storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
     // bit 32 = sign 0:positive, 1:negative
     // bit 31 = 0:divide, 1:multiply
     // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
@@ -19,7 +19,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
         goto RET_SN;
 
     int m = 0;
-    calculated_number n = value, factor = 10;
+    NETDATA_DOUBLE n = value, factor = 10;
 
     // if the value is negative
     // add the sign bit and make it positive
@@ -36,7 +36,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
     // make its integer part fit in 0x00ffffff
     // by dividing it by 10 up to 7 times
     // and increasing the multiplier
-    while(m < 7 && n > (calculated_number)0x00ffffff) {
+    while(m < 7 && n > (NETDATA_DOUBLE)0x00ffffff) {
         n /= factor;
         m++;
     }
@@ -46,9 +46,9 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
         // so we add a multiplier to unpack it
         r += (1 << 30) + (m << 27); // the multiplier m
 
-        if(n > (calculated_number)0x00ffffff) {
+        if(n > (NETDATA_DOUBLE)0x00ffffff) {
             #ifdef NETDATA_INTERNAL_CHECKS
-            error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
+            error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value);
             #endif
             r += 0x00ffffff;
             goto RET_SN;
@@ -60,12 +60,12 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
         // while the value is below 0x0019999e we can
         // multiply it by 10, up to 7 times, increasing
         // the multiplier
-        while(m < 7 && n < (calculated_number)0x0019999e) {
+        while(m < 7 && n < (NETDATA_DOUBLE)0x0019999e) {
             n *= 10;
             m++;
         }
 
-        if (unlikely(n > (calculated_number) (0x00ffffff))) {
+        if (unlikely(n > (NETDATA_DOUBLE) (0x00ffffff))) {
             n /= 10;
             m--;
         }
@@ -90,7 +90,7 @@ RET_SN:
 }
 
 // Lookup table to make storage number unpacking efficient.
-calculated_number unpack_storage_number_lut10x[4 * 8];
+NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
 
 __attribute__((constructor)) void initialize_lut(void) {
     // The lookup table is partitioned in 4 subtables based on the
@@ -107,7 +107,7 @@ __attribute__((constructor)) void initialize_lut(void) {
 }
 
 /*
-int print_calculated_number(char *str, calculated_number value)
+int print_netdata_double(char *str, NETDATA_DOUBLE value)
 {
     char *wstr = str;
 
@@ -117,9 +117,9 @@ int print_calculated_number(char *str, calculated_number value)
 #ifdef STORAGE_WITH_MATH
     // without llrintl() there are rounding problems
     // for example 0.9 becomes 0.89
-    unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000);
+    unsigned long long uvalue = (unsigned long long int) llrintl(value * (NETDATA_DOUBLE)100000);
 #else
-    unsigned long long uvalue = value * (calculated_number)100000;
+    unsigned long long uvalue = value * (NETDATA_DOUBLE)100000;
 #endif
 
     wstr = print_number_llu_r_smart(str, uvalue);
@@ -163,8 +163,8 @@ int print_calculated_number(char *str, calculated_number value)
 }
 */
 
-int print_calculated_number(char *str, calculated_number value) {
-    // info("printing number " CALCULATED_NUMBER_FORMAT, value);
+int print_netdata_double(char *str, NETDATA_DOUBLE value) {
+    // info("printing number " NETDATA_DOUBLE_FORMAT, value);
     char integral_str[50], fractional_str[50];
 
     char *wstr = str;
@@ -174,22 +174,22 @@ int print_calculated_number(char *str, calculated_number value) {
         value = -value;
     }
 
-    calculated_number integral, fractional;
+    NETDATA_DOUBLE integral, fractional;
 
 #ifdef STORAGE_WITH_MATH
-    fractional = calculated_number_modf(value, &integral) * 10000000.0;
+    fractional = modfndd(value, &integral) * 10000000.0;
 #else
     fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
 #endif
 
     unsigned long long integral_int = (unsigned long long)integral;
-    unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional);
+    unsigned long long fractional_int = (unsigned long long)llrintndd(fractional);
     if(unlikely(fractional_int >= 10000000)) {
         integral_int += 1;
         fractional_int -= 10000000;
     }
 
-    // info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
+    // info("integral " NETDATA_DOUBLE_FORMAT " (%llu), fractional " NETDATA_DOUBLE_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
 
     char *istre;
     if(unlikely(integral_int == 0)) {
diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h
index 64d59c17c2..722d35d386 100644
--- a/libnetdata/storage_number/storage_number.h
+++ b/libnetdata/storage_number/storage_number.h
@@ -6,68 +6,60 @@
 #include <math.h>
 #include "../libnetdata.h"
 
-#ifdef NETDATA_WITHOUT_LONG_DOUBLE
-
-#define powl pow
-#define modfl modf
-#define llrintl llrint
-#define roundl round
-#define sqrtl sqrt
-#define copysignl copysign
-#define strtold strtod
-
-typedef double calculated_number;
-#define CALCULATED_NUMBER_FORMAT "%0.7f"
-#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f"
-#define CALCULATED_NUMBER_FORMAT_AUTO "%f"
-
-#define LONG_DOUBLE_MODIFIER "f"
-typedef double LONG_DOUBLE;
-
-#define CALCULATED_NUMBER_MAX DBL_MAX
-
-#else // NETDATA_WITHOUT_LONG_DOUBLE
-
-typedef long double calculated_number;
-#define CALCULATED_NUMBER_FORMAT "%0.7Lf"
-#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf"
-#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf"
-
-#define LONG_DOUBLE_MODIFIER "Lf"
-typedef long double LONG_DOUBLE;
-
-#define CALCULATED_NUMBER_MAX LDBL_MAX
-
-#endif // NETDATA_WITHOUT_LONG_DOUBLE
-
-//typedef long long calculated_number;
-//#define CALCULATED_NUMBER_FORMAT "%lld"
+#ifdef NETDATA_WITH_LONG_DOUBLE
+
+typedef long double NETDATA_DOUBLE;
+#define NETDATA_DOUBLE_FORMAT "%0.7Lf"
+#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0Lf"
+#define NETDATA_DOUBLE_FORMAT_AUTO "%Lf"
+#define NETDATA_DOUBLE_MODIFIER "Lf"
+
+#define NETDATA_DOUBLE_MAX LDBL_MAX
+
+#define strtondd(s, endptr) strtold(s, endptr)
+#define powndd(x, y) powl(x, y)
+#define llrintndd(x) llrintl(x)
+#define roundndd(x) roundl(x)
+#define sqrtndd(x) sqrtl(x)
+#define copysignndd(x, y) copysignl(x, y)
+#define modfndd(x, y) modfl(x, y)
+#define fabsndd(x) fabsl(x)
+
+#else // NETDATA_WITH_LONG_DOUBLE
+
+typedef double NETDATA_DOUBLE;
+#define NETDATA_DOUBLE_FORMAT "%0.7f"
+#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0f"
+#define NETDATA_DOUBLE_FORMAT_AUTO "%f"
+#define NETDATA_DOUBLE_MODIFIER "f"
+
+#define NETDATA_DOUBLE_MAX DBL_MAX
+
+#define strtondd(s, endptr) strtod(s, endptr)
+#define powndd(x, y) pow(x, y)
+#define llrintndd(x) llrint(x)
+#define roundndd(x) round(x)
+#define sqrtndd(x) sqrt(x)
+#define copysignndd(x, y) copysign(x, y)
+#define modfndd(x, y) modf(x, y)
+#define fabsndd(x) fabs(x)
+
+#endif // NETDATA_WITH_LONG_DOUBLE
 
 typedef long long collected_number;
 #define COLLECTED_NUMBER_FORMAT "%lld"
 
-/*
-typedef long double collected_number;
-#define COLLECTED_NUMBER_FORMAT "%0.7Lf"
-*/
-
-#define calculated_number_modf(x, y) modfl(x, y)
-#define calculated_number_llrint(x) llrintl(x)
-#define calculated_number_round(x) roundl(x)
-#define calculated_number_fabs(x) fabsl(x)
-#define calculated_number_pow(x, y) powl(x, y)
-#define calculated_number_epsilon (calculated_number)0.0000001
-
-#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)
+#define epsilonndd (NETDATA_DOUBLE)0.0000001
+#define considered_equal_ndd(a, b) (fabsndd((a) - (b)) < epsilonndd)
 
 #if defined(HAVE_ISFINITE) || defined(isfinite)
 // The isfinite() macro shall determine whether its argument has a
 // finite value (zero, subnormal, or normal, and not infinite or NaN).
-#define calculated_number_isnumber(a) (isfinite(a))
+#define netdata_double_isnumber(a) (isfinite(a))
 #elif defined(HAVE_FINITE) || defined(finite)
-#define calculated_number_isnumber(a) (finite(a))
+#define netdata_double_isnumber(a) (finite(a))
 #else
-#define calculated_number_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
+#define netdata_double_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
 #endif
 
 typedef uint32_t storage_number;
@@ -95,10 +87,10 @@ typedef enum {
 #define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)
 #define did_storage_number_reset(value)  ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)
 
-storage_number pack_storage_number(calculated_number value, SN_FLAGS flags);
-static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const));
+storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags);
+static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attribute__((const));
 
-int print_calculated_number(char *str, calculated_number value);
+int print_netdata_double(char *str, NETDATA_DOUBLE value);
 
 //                                          sign       div/mul    <--- multiplier / divider --->     10/100       RESET      EXISTS     VALUE
 #define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
@@ -115,8 +107,8 @@ int print_calculated_number(char *str, calculated_number value);
 #define MAX_INCREMENTAL_PERCENT_RATE 10
 
 
-static inline calculated_number unpack_storage_number(storage_number value) {
-    extern calculated_number unpack_storage_number_lut10x[4 * 8];
+static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) {
+    extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
 
     if(unlikely(value == SN_EMPTY_SLOT))
         return NAN;
@@ -145,11 +137,82 @@ static inline calculated_number unpack_storage_number(storage_number value) {
     // bit 24 to bit 1 = the value, so remove all other bits
     value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
 
-    calculated_number n = value;
+    NETDATA_DOUBLE n = value;
 
     // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n);
 
     return sign * unpack_storage_number_lut10x[(factor * 16) + (exp * 8) + mul] * n;
 }
 
+static inline NETDATA_DOUBLE str2ndd(const char *s, char **endptr) {
+    int negative = 0;
+    const char *start = s;
+    unsigned long long integer_part = 0;
+    unsigned long decimal_part = 0;
+    size_t decimal_digits = 0;
+
+    switch(*s) {
+        case '-':
+            s++;
+            negative = 1;
+            break;
+
+        case '+':
+            s++;
+            break;
+
+        case 'n':
+            if(s[1] == 'a' && s[2] == 'n') {
+                if(endptr) *endptr = (char *)&s[3];
+                return NAN;
+            }
+            break;
+
+        case 'i':
+            if(s[1] == 'n' && s[2] == 'f') {
+                if(endptr) *endptr = (char *)&s[3];
+                return INFINITY;
+            }
+            break;
+
+        default:
+            break;
+    }
+
+    while (*s >= '0' && *s <= '9') {
+        integer_part = (integer_part * 10) + (*s - '0');
+        s++;
+    }
+
+    if(unlikely(*s == '.')) {
+        decimal_part = 0;
+        s++;
+
+        while (*s >= '0' && *s <= '9') {
+            decimal_part = (decimal_part * 10) + (*s - '0');
+            s++;
+            decimal_digits++;
+        }
+    }
+
+    if(unlikely(*s == 'e' || *s == 'E'))
+        return strtondd(start, endptr);
+
+    if(unlikely(endptr))
+        *endptr = (char *)s;
+
+    if(unlikely(negative)) {
+        if(unlikely(decimal_digits))
+            return -((NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits));
+        else
+            return -((NETDATA_DOUBLE)integer_part);
+    }
+    else {
+        if(unlikely(decimal_digits))
+            return (NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits);
+        else
+            return (NETDATA_DOUBLE)integer_part;
+    }
+}
+
 #endif /* NETDATA_STORAGE_NUMBER_H */
diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c
index f90521cabe..19309e5c26 100644
--- a/libnetdata/storage_number/tests/test_storage_number.c
+++ b/libnetdata/storage_number/tests/test_storage_number.c
@@ -11,34 +11,34 @@ static void test_number_printing(void **state)
 
     char value[50];
 
-    print_calculated_number(value, 0);
+    print_netdata_double(value, 0);
     assert_string_equal(value, "0");
 
-    print_calculated_number(value, 0.0000001);
+    print_netdata_double(value, 0.0000001);
     assert_string_equal(value, "0.0000001");
 
-    print_calculated_number(value, 0.00000009);
+    print_netdata_double(value, 0.00000009);
     assert_string_equal(value, "0.0000001");
 
-    print_calculated_number(value, 0.000000001);
+    print_netdata_double(value, 0.000000001);
     assert_string_equal(value, "0");
 
-    print_calculated_number(value, 99.99999999999999999);
+    print_netdata_double(value, 99.99999999999999999);
     assert_string_equal(value, "100");
 
-    print_calculated_number(value, -99.99999999999999999);
+    print_netdata_double(value, -99.99999999999999999);
     assert_string_equal(value, "-100");
 
-    print_calculated_number(value, 123.4567890123456789);
+    print_netdata_double(value, 123.4567890123456789);
     assert_string_equal(value, "123.456789");
 
-    print_calculated_number(value, 9999.9999999);
+    print_netdata_double(value, 9999.9999999);
     assert_string_equal(value, "9999.9999999");
 
-    print_calculated_number(value, -9999.9999999);
+    print_netdata_double(value, -9999.9999999);
     assert_string_equal(value, "-9999.9999999");
 
-    print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
+    print_netdata_double(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
     assert_string_equal(value, "16.77722");
 }