JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)

* first work on standardizing json formatting

* renamed old grouping to time_grouping and added group_by

* add dummy functions to enable compilation

* buffer json api work

* jsonwrap opening with buffer_json_X() functions

* cleanup

* storage for quotes

* optimize buffer printing for both numbers and strings

* removed ; from define

* contexts json generation using the new json functions

* fix buffer overflow at unit test

* weights endpoint using new json api

* fixes to weights endpoint

* check buffer overflow on all buffer functions

* do synchronous queries for weights

* buffer_flush() now resets json state too

* content type typedef

* print double values that are above the max 64-bit value

* str2ndd() can now parse values above UINT64_MAX

* faster number parsing by avoiding double calculations as much as possible

* faster number parsing

* faster hex parsing

* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers

* full printing and parsing without using library functions - and related unit tests

* added IEEE754 streaming capability to enable streaming of double values in hex

* streaming and replication to transfer all values in hex

* use our own str2ndd for set2

* remove subnormal check from ieee

* base64 encoding for numbers, instead of hex

* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision

* str2ndd_encoded() parses all encoding formats, including integers

* prevent uninitialized use

* /api/v1/info using the new json API

* Fix error when compiling with --disable-ml

* Remove redundant 'buffer_unittest' declaration

* Fix formatting

* Fix formatting

* Fix formatting

* fix buffer unit test

* apps.plugin using the new JSON API

* make sure the metrics registry does not accept negative timestamps

* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache

* Fix more formatting

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 19 insertions, 9 deletions

diff --git a/daemon/main.c b/daemon/main.c
index 009cbe6ed1..fd9c1c922f 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -13,6 +13,7 @@ int netdata_zero_metrics_enabled;
 int netdata_anonymous_statistics_enabled;
 
 int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;
+bool ieee754_doubles = false;
 
 struct netdata_static_thread *static_threads;
 
@@ -1314,6 +1315,7 @@ void post_conf_load(char **user)
         prev_msg = msg;                                 \
     }
 
+int buffer_unittest(void);
 int pgc_unittest(void);
 int mrg_unittest(void);
 int julytest(void);
@@ -1447,6 +1449,8 @@ int main(int argc, char **argv) {
                                 return 1;
                             if (unit_test_str2ld())
                                 return 1;
+                            if (buffer_unittest())
+                                return 1;
                             if (unit_test_bitmap256())
                                 return 1;
                             // No call to load the config file on this code-path
@@ -1483,15 +1487,6 @@ int main(int argc, char **argv) {
                         else if(strcmp(optarg, "escapetest") == 0) {
                             return command_argument_sanitization_tests();
                         }
-#ifdef ENABLE_DBENGINE
-                        else if(strcmp(optarg, "mctest") == 0) {
-                            unittest_running = true;
-                            return mc_unittest();
-                        }
-                        else if(strcmp(optarg, "ctxtest") == 0) {
-                            unittest_running = true;
-                            return ctx_unittest();
-                        }
                         else if(strcmp(optarg, "dicttest") == 0) {
                             unittest_running = true;
                             return dictionary_unittest(10000);
@@ -1508,6 +1503,19 @@ int main(int argc, char **argv) {
                             unittest_running = true;
                             return rrdlabels_unittest();
                         }
+                        else if(strcmp(optarg, "buffertest") == 0) {
+                            unittest_running = true;
+                            return buffer_unittest();
+                        }
+#ifdef ENABLE_DBENGINE
+                        else if(strcmp(optarg, "mctest") == 0) {
+                            unittest_running = true;
+                            return mc_unittest();
+                        }
+                        else if(strcmp(optarg, "ctxtest") == 0) {
+                            unittest_running = true;
+                            return ctx_unittest();
+                        }
                         else if(strcmp(optarg, "metatest") == 0) {
                             unittest_running = true;
                             return metadata_unittest();
@@ -1878,6 +1886,8 @@ int main(int argc, char **argv) {
         // initialize the log files
         open_all_log_files();
 
+        ieee754_doubles = is_system_ieee754_double();
+
         aral_judy_init();
 
         get_system_timezone();