netdata doubles (#13217)

* netdata doubles

* fix cmocka test

* fix cmocka test again

* fix left-overs of long double to NETDATA_DOUBLE

* RRDDIM detached from disk representation; db settings in [db] section of netdata.conf

* update the memory before saving

* rrdset is now detached from file structures too

* on memory mode map, update the memory mapped structures on every iteration

* allow RRD_ID_LENGTH_MAX to be changed

* granularity secs, back to update every

* fix formatting

* more formatting

10 files changed, 34 insertions, 33 deletions

diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 0feb38d2b3..9683ca50b8 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -269,7 +269,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st);
 
 extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options);
 
-calculated_number exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE
+exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp);
diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 9dbaf21dd4..f5966bf489 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -173,14 +173,14 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
         RRD_ID_LENGTH_MAX);
 
     time_t last_t;
-    calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+    NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
 
     if(isnan(value))
         return 0;
 
     buffer_sprintf(
         instance->buffer,
-        "%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
+        "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
         instance->config.prefix,
         (host == localhost) ? instance->config.hostname : host->hostname,
         chart_name,
diff --git a/exporting/json/json.c b/exporting/json/json.c
index c8db216e4a..91f21e61d1 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -224,7 +224,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
     RRDHOST *host = st->rrdhost;
 
     time_t last_t;
-    calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+    NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
 
     if(isnan(value))
         return 0;
@@ -265,7 +265,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
 
         "\"id\":\"%s\","
         "\"name\":\"%s\","
-        "\"value\":" CALCULATED_NUMBER_FORMAT ","
+        "\"value\":" NETDATA_DOUBLE_FORMAT ","
 
         "\"timestamp\": %llu}",
 
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index d6ce98ce45..268a437372 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -225,14 +225,14 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
         RRD_ID_LENGTH_MAX);
 
     time_t last_t;
-    calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+    NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
 
     if(isnan(value))
         return 0;
 
     buffer_sprintf(
         instance->buffer,
-        "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n",
+        "put %s.%s.%s %llu " NETDATA_DOUBLE_FORMAT " host=%s%s%s%s\n",
         instance->config.prefix,
         chart_name,
         dimension_name,
@@ -368,7 +368,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
         RRD_ID_LENGTH_MAX);
 
     time_t last_t;
-    calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+    NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
 
     if(isnan(value))
         return 0;
@@ -381,7 +381,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
         "{"
         "\"metric\":\"%s.%s.%s\","
         "\"timestamp\":%llu,"
-        "\"value\":"CALCULATED_NUMBER_FORMAT","
+        "\"value\":" NETDATA_DOUBLE_FORMAT ","
         "\"tags\":{"
         "\"host\":\"%s%s%s\"%s"
         "}"
diff --git a/exporting/process_data.c b/exporting/process_data.c
index 83f0e012ea..f70b39294c 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -64,7 +64,7 @@ int mark_scheduled_instances(struct engine *engine)
  * @param last_timestamp the timestamp that should be reported to the exporting connector instance.
  * @return Returns the value, calculated over the given period.
  */
-calculated_number exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp)
@@ -123,15 +123,15 @@ calculated_number exporting_calculate_value_from_stored_data(
     *last_timestamp = before;
 
     size_t counter = 0;
-    calculated_number sum = 0;
-    calculated_number value;
+    NETDATA_DOUBLE sum = 0;
+    NETDATA_DOUBLE value;
 
     for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
         time_t curr_t, end_t;
         SN_FLAGS flags;
         value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
 
-        if (unlikely(!calculated_number_isnumber(value))) {
+        if (unlikely(!netdata_double_isnumber(value))) {
             // not collected
             continue;
         }
@@ -156,7 +156,7 @@ calculated_number exporting_calculate_value_from_stored_data(
     if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM))
         return sum;
 
-    return sum / (calculated_number)counter;
+    return sum / (NETDATA_DOUBLE)counter;
 }
 
 /**
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index 95795742b8..1a9c72e8a0 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -362,7 +362,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
             }
         }
 
-        calculated_number value = rrdvar2number(rv);
+        NETDATA_DOUBLE value = rrdvar2number(rv);
         if (isnan(value) || isinf(value)) {
             if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
                 buffer_sprintf(
@@ -383,7 +383,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
         if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
             buffer_sprintf(
                 opts->wb,
-                "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
+                "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
                 opts->prefix,
                 opts->name,
                 label_pre,
@@ -394,7 +394,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
         else
             buffer_sprintf(
                 opts->wb,
-                "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n",
+                "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT "\n",
                 opts->prefix,
                 opts->name,
                 label_pre,
@@ -483,9 +483,9 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
     if (prometheus_collector)
         buffer_sprintf(
             wb,
-            CALCULATED_NUMBER_FORMAT,
-            (calculated_number)p->rd->last_collected_value * (calculated_number)p->rd->multiplier /
-                (calculated_number)p->rd->divisor);
+            NETDATA_DOUBLE_FORMAT,
+            (NETDATA_DOUBLE)p->rd->last_collected_value * (NETDATA_DOUBLE)p->rd->multiplier /
+                (NETDATA_DOUBLE)p->rd->divisor);
     else
         buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value);
 
@@ -732,7 +732,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
 
                         time_t first_time = instance->after;
                         time_t last_time = instance->before;
-                        calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
+                        NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
 
                         if (!isnan(value) && !isinf(value)) {
                             if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@@ -764,7 +764,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
                                 buffer_sprintf(
                                     wb,
-                                    "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
+                                    "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
                                     " %llu\n",
                                     prefix,
                                     context,
@@ -779,7 +779,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                             else
                                 buffer_sprintf(
                                     wb,
-                                    "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
+                                    "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
                                     "\n",
                                     prefix,
                                     context,
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index ac00d3b50e..f6abaab36c 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -294,7 +294,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
             // we need average or sum of the data
 
             time_t last_t = instance->before;
-            calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+            NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
 
             if (!isnan(value) && !isinf(value)) {
                 if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)
diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c
index b8c9f37560..da4f40f0d3 100644
--- a/exporting/tests/exporting_doubles.c
+++ b/exporting/tests/exporting_doubles.c
@@ -52,11 +52,11 @@ int __wrap_mark_scheduled_instances(struct engine *engine)
     return mock_type(int);
 }
 
-calculated_number __real_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp);
-calculated_number __wrap_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp)
@@ -67,7 +67,7 @@ calculated_number __wrap_exporting_calculate_value_from_stored_data(
     *last_timestamp = 15052;
 
     function_called();
-    return mock_type(calculated_number);
+    return mock_type(NETDATA_DOUBLE);
 }
 
 int __real_prepare_buffers(struct engine *engine);
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index 8838d6574a..066f52f215 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -177,7 +177,7 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
     return RRD_MEMORY_MODE_NONE_NAME;
 }
 
-calculated_number rrdvar2number(RRDVAR *rv)
+NETDATA_DOUBLE rrdvar2number(RRDVAR *rv)
 {
     (void)rv;
     return 0;
@@ -230,7 +230,7 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
     return mock_type(int);
 }
 
-calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
+NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
 {
     (void)handle;
     (void)start_time;
@@ -239,7 +239,7 @@ calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *ha
 
 
     function_called();
-    return mock_type(calculated_number);
+    return mock_type(NETDATA_DOUBLE);
 }
 
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h
index 7431e109ba..298a77a9fd 100644
--- a/exporting/tests/test_exporting_engine.h
+++ b/exporting/tests/test_exporting_engine.h
@@ -61,7 +61,7 @@ time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
 time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
 void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
 int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
-calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
+NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
 void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
 
 // -----------------------------------------------------------------------
@@ -88,11 +88,11 @@ int __wrap_init_connectors(struct engine *engine);
 int __real_mark_scheduled_instances(struct engine *engine);
 int __wrap_mark_scheduled_instances(struct engine *engine);
 
-calculated_number __real_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp);
-calculated_number __wrap_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
     struct instance *instance,
     RRDDIM *rd,
     time_t *last_timestamp);