JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)

* first work on standardizing json formatting

* renamed old grouping to time_grouping and added group_by

* add dummy functions to enable compilation

* buffer json api work

* jsonwrap opening with buffer_json_X() functions

* cleanup

* storage for quotes

* optimize buffer printing for both numbers and strings

* removed ; from define

* contexts json generation using the new json functions

* fix buffer overflow at unit test

* weights endpoint using new json api

* fixes to weights endpoint

* check buffer overflow on all buffer functions

* do synchronous queries for weights

* buffer_flush() now resets json state too

* content type typedef

* print double values that are above the max 64-bit value

* str2ndd() can now parse values above UINT64_MAX

* faster number parsing by avoiding double calculations as much as possible

* faster number parsing

* faster hex parsing

* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers

* full printing and parsing without using library functions - and related unit tests

* added IEEE754 streaming capability to enable streaming of double values in hex

* streaming and replication to transfer all values in hex

* use our own str2ndd for set2

* remove subnormal check from ieee

* base64 encoding for numbers, instead of hex

* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision

* str2ndd_encoded() parses all encoding formats, including integers

* prevent uninitialized use

* /api/v1/info using the new json API

* Fix error when compiling with --disable-ml

* Remove redundant 'buffer_unittest' declaration

* Fix formatting

* Fix formatting

* Fix formatting

* fix buffer unit test

* apps.plugin using the new JSON API

* make sure the metrics registry does not accept negative timestamps

* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache

* Fix more formatting

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

7 files changed, 27 insertions, 30 deletions

diff --git a/ml/Config.cc b/ml/Config.cc
index ba3a614452..cdc43d1655 100644
--- a/ml/Config.cc
+++ b/ml/Config.cc
@@ -99,7 +99,8 @@ void Config::readMLConfig(void) {
     Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
 
     Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
-    Cfg.AnomalyDetectionGroupingMethod = web_client_api_request_v1_data_group(AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
+    Cfg.AnomalyDetectionGroupingMethod = time_grouping_parse(
+            AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
     Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
 
     Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
diff --git a/ml/Config.h b/ml/Config.h
index f10e114926..26ceaf3004 100644
--- a/ml/Config.h
+++ b/ml/Config.h
@@ -29,7 +29,7 @@ public:
     double DimensionAnomalyScoreThreshold;
 
     double HostAnomalyRateThreshold;
-    RRDR_GROUPING AnomalyDetectionGroupingMethod;
+    RRDR_TIME_GROUPING AnomalyDetectionGroupingMethod;
     time_t AnomalyDetectionQueryDuration;
 
     bool StreamADCharts;
diff --git a/ml/Host.cc b/ml/Host.cc
index a5f276a807..e22580be9a 100644
--- a/ml/Host.cc
+++ b/ml/Host.cc
@@ -19,30 +19,30 @@ void Host::removeChart(Chart *C) {
     Charts.erase(C->getRS());
 }
 
-void Host::getConfigAsJson(nlohmann::json &Json) const {
-    Json["version"] = 1;
+void Host::getConfigAsJson(BUFFER *wb) const {
+    buffer_json_member_add_uint64(wb, "version", 1);
 
-    Json["enabled"] = Cfg.EnableAnomalyDetection;
+    buffer_json_member_add_boolean(wb, "enabled", Cfg.EnableAnomalyDetection);
 
-    Json["min-train-samples"] = Cfg.MinTrainSamples;
-    Json["max-train-samples"] = Cfg.MaxTrainSamples;
-    Json["train-every"] = Cfg.TrainEvery;
+    buffer_json_member_add_uint64(wb, "min-train-samples", Cfg.MinTrainSamples);
+    buffer_json_member_add_uint64(wb, "max-train-samples", Cfg.MaxTrainSamples);
+    buffer_json_member_add_uint64(wb, "train-every", Cfg.TrainEvery);
 
-    Json["diff-n"] = Cfg.DiffN;
-    Json["smooth-n"] = Cfg.SmoothN;
-    Json["lag-n"] = Cfg.LagN;
+    buffer_json_member_add_uint64(wb, "diff-n", Cfg.DiffN);
+    buffer_json_member_add_uint64(wb, "smooth-n", Cfg.SmoothN);
+    buffer_json_member_add_uint64(wb, "lag-n", Cfg.LagN);
 
-    Json["random-sampling-ratio"] = Cfg.RandomSamplingRatio;
-    Json["max-kmeans-iters"] = Cfg.MaxKMeansIters;
+    buffer_json_member_add_double(wb, "random-sampling-ratio", Cfg.RandomSamplingRatio);
+    buffer_json_member_add_uint64(wb, "max-kmeans-iters", Cfg.MaxKMeansIters);
 
-    Json["dimension-anomaly-score-threshold"] = Cfg.DimensionAnomalyScoreThreshold;
+    buffer_json_member_add_double(wb, "dimension-anomaly-score-threshold", Cfg.DimensionAnomalyScoreThreshold);
 
-    Json["host-anomaly-rate-threshold"] = Cfg.HostAnomalyRateThreshold;
-    Json["anomaly-detection-grouping-method"] = group_method2string(Cfg.AnomalyDetectionGroupingMethod);
-    Json["anomaly-detection-query-duration"] = Cfg.AnomalyDetectionQueryDuration;
+    buffer_json_member_add_double(wb, "host-anomaly-rate-threshold", Cfg.HostAnomalyRateThreshold);
+    buffer_json_member_add_string(wb, "anomaly-detection-grouping-method", time_grouping_method2string(Cfg.AnomalyDetectionGroupingMethod));
+    buffer_json_member_add_time_t(wb, "anomaly-detection-query-duration", Cfg.AnomalyDetectionQueryDuration);
 
-    Json["hosts-to-skip"] = Cfg.HostsToSkip;
-    Json["charts-to-skip"] = Cfg.ChartsToSkip;
+    buffer_json_member_add_string(wb, "hosts-to-skip", Cfg.HostsToSkip.c_str());
+    buffer_json_member_add_string(wb, "charts-to-skip", Cfg.ChartsToSkip.c_str());
 }
 
 void Host::getModelsAsJson(nlohmann::json &Json) {
diff --git a/ml/Host.h b/ml/Host.h
index 289cb5ab7e..c6e2e7080c 100644
--- a/ml/Host.h
+++ b/ml/Host.h
@@ -34,7 +34,7 @@ public:
     void addChart(Chart *C);
     void removeChart(Chart *C);
 
-    void getConfigAsJson(nlohmann::json &Json) const;
+    void getConfigAsJson(BUFFER *wb) const;
     void getModelsAsJson(nlohmann::json &Json);
     void getDetectionInfoAsJson(nlohmann::json &Json) const;
 
diff --git a/ml/ml-dummy.c b/ml/ml-dummy.c
index 966602d2c8..c3dcb93370 100644
--- a/ml/ml-dummy.c
+++ b/ml/ml-dummy.c
@@ -47,9 +47,9 @@ void ml_stop_anomaly_detection_threads(RRDHOST *RH) {
     UNUSED(RH);
 }
 
-char *ml_get_host_info(RRDHOST *RH) {
+void ml_get_host_info(RRDHOST *RH, BUFFER *wb) {
     (void) RH;
-    return NULL;
+    (void) wb;
 }
 
 char *ml_get_host_runtime_info(RRDHOST *RH) {
diff --git a/ml/ml.cc b/ml/ml.cc
index 0b053f98a1..cb9783cf7b 100644
--- a/ml/ml.cc
+++ b/ml/ml.cc
@@ -108,17 +108,13 @@ void ml_dimension_delete(RRDDIM *RD) {
     RD->ml_dimension = nullptr;
 }
 
-char *ml_get_host_info(RRDHOST *RH) {
-    nlohmann::json ConfigJson;
-
+void ml_get_host_info(RRDHOST *RH, BUFFER *wb) {
     if (RH && RH->ml_host) {
         Host *H = reinterpret_cast<Host *>(RH->ml_host);
-        H->getConfigAsJson(ConfigJson);
+        H->getConfigAsJson(wb);
     } else {
-        ConfigJson["enabled"] = false;
+        buffer_json_member_add_boolean(wb, "enabled", false);
     }
-
-    return strdupz(ConfigJson.dump(2, '\t').c_str());
 }
 
 char *ml_get_host_runtime_info(RRDHOST *RH) {
diff --git a/ml/ml.h b/ml/ml.h
index 320fc0b2f9..a6738294bd 100644
--- a/ml/ml.h
+++ b/ml/ml.h
@@ -33,7 +33,7 @@ void ml_start_anomaly_detection_threads(RRDHOST *RH);
 void ml_stop_anomaly_detection_threads(RRDHOST *RH);
 void ml_cancel_anomaly_detection_threads(RRDHOST *RH);
 
-char *ml_get_host_info(RRDHOST *RH);
+void ml_get_host_info(RRDHOST *RH, BUFFER *wb);
 char *ml_get_host_runtime_info(RRDHOST *RH);
 char *ml_get_host_models(RRDHOST *RH);