JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)

* first work on standardizing json formatting

* renamed old grouping to time_grouping and added group_by

* add dummy functions to enable compilation

* buffer json api work

* jsonwrap opening with buffer_json_X() functions

* cleanup

* storage for quotes

* optimize buffer printing for both numbers and strings

* removed ; from define

* contexts json generation using the new json functions

* fix buffer overflow at unit test

* weights endpoint using new json api

* fixes to weights endpoint

* check buffer overflow on all buffer functions

* do synchronous queries for weights

* buffer_flush() now resets json state too

* content type typedef

* print double values that are above the max 64-bit value

* str2ndd() can now parse values above UINT64_MAX

* faster number parsing by avoiding double calculations as much as possible

* faster number parsing

* faster hex parsing

* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers

* full printing and parsing without using library functions - and related unit tests

* added IEEE754 streaming capability to enable streaming of double values in hex

* streaming and replication to transfer all values in hex

* use our own str2ndd for set2

* remove subnormal check from ieee

* base64 encoding for numbers, instead of hex

* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision

* str2ndd_encoded() parses all encoding formats, including integers

* prevent uninitialized use

* /api/v1/info using the new json API

* Fix error when compiling with --disable-ml

* Remove redundant 'buffer_unittest' declaration

* Fix formatting

* Fix formatting

* Fix formatting

* fix buffer unit test

* apps.plugin using the new JSON API

* make sure the metrics registry does not accept negative timestamps

* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache

* Fix more formatting

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

7 files changed, 1252 insertions, 596 deletions

diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
index 7f4c6c53d9..6332fa174d 100644
--- a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
+++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
@@ -9,7 +9,7 @@ inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *va
     (void)hash;
 
     register unsigned long long *d = dst;
-    *d = str2ull(value);
+    *d = str2ull(value, NULL);
     // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, *d);
 }
 
diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c
index 6cd6ae38a5..6e27bcc79e 100644
--- a/libnetdata/buffer/buffer.c
+++ b/libnetdata/buffer/buffer.c
@@ -2,39 +2,16 @@
 
 #include "../libnetdata.h"
 
-#define BUFFER_OVERFLOW_EOF "EOF"
-
 static inline void buffer_overflow_init(BUFFER *b)
 {
     b->buffer[b->size] = '\0';
     strcpy(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF);
 }
 
-#ifdef NETDATA_INTERNAL_CHECKS
-#define buffer_overflow_check(b) _buffer_overflow_check(b, __FILE__, __FUNCTION__, __LINE__)
-#else
-#define buffer_overflow_check(b)
-#endif
-
-static inline void _buffer_overflow_check(BUFFER *b, const char *file, const char *function, const unsigned long line)
-{
-    if(b->len > b->size) {
-        error("BUFFER: length %zu is above size %zu, at line %lu, at function %s() of file '%s'.", b->len, b->size, line, function, file);
-        b->len = b->size;
-    }
-
-    if(b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF) != 0) {
-        error("BUFFER: detected overflow at line %lu, at function %s() of file '%s'.", line, function, file);
-        buffer_overflow_init(b);
-    }
-}
-
-
-void buffer_reset(BUFFER *wb)
-{
+void buffer_reset(BUFFER *wb) {
     buffer_flush(wb);
 
-    wb->contenttype = CT_TEXT_PLAIN;
+    wb->content_type = CT_TEXT_PLAIN;
     wb->options = 0;
     wb->date = 0;
     wb->expires = 0;
@@ -52,8 +29,7 @@ const char *buffer_tostring(BUFFER *wb)
     return(wb->buffer);
 }
 
-void buffer_char_replace(BUFFER *wb, char from, char to)
-{
+void buffer_char_replace(BUFFER *wb, char from, char to) {
     char *s = wb->buffer, *end = &wb->buffer[wb->len];
 
     while(s != end) {
@@ -64,189 +40,6 @@ void buffer_char_replace(BUFFER *wb, char from, char to)
     buffer_overflow_check(wb);
 }
 
-// This trick seems to give an 80% speed increase in 32bit systems
-// print_number_llu_r() will just print the digits up to the
-// point the remaining value fits in 32 bits, and then calls
-// print_number_lu_r() to print the rest with 32 bit arithmetic.
-
-inline char *print_number_lu_r(char *str, unsigned long uvalue) {
-    char *wstr = str;
-
-    // print each digit
-    do *wstr++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10);
-    return wstr;
-}
-
-inline char *print_number_llu_r(char *str, unsigned long long uvalue) {
-    char *wstr = str;
-
-    // print each digit
-    do *wstr++ = (char)('0' + (uvalue % 10)); while((uvalue /= 10) && uvalue > (unsigned long long)0xffffffff);
-    if(uvalue) return print_number_lu_r(wstr, uvalue);
-    return wstr;
-}
-
-inline char *print_number_llu_r_smart(char *str, unsigned long long uvalue) {
-    switch (sizeof(void *)) {
-    case 4:
-        str = (uvalue > (unsigned long long) 0xffffffff) ? print_number_llu_r(str, uvalue) :
-                                                           print_number_lu_r(str, uvalue);
-        break;
-    case 8:
-        do {
-            *str++ = (char) ('0' + (uvalue % 10));
-        } while (uvalue /= 10);
-        break;
-    default:
-        fatal("Netdata supports only 32-bit & 64-bit systems.");
-    }
-
-    return str;
-}
-
-void buffer_print_llu(BUFFER *wb, unsigned long long uvalue)
-{
-    buffer_need_bytes(wb, 50);
-
-    switch(uvalue) {
-        case 0:
-            buffer_fast_strcat(wb, "0", 1);
-            return;
-
-        case 1:
-            buffer_fast_strcat(wb, "1", 1);
-            return;
-
-        case 5:
-            buffer_fast_strcat(wb, "5", 1);
-            return;
-
-        case 10:
-            buffer_fast_strcat(wb, "10", 2);
-            return;
-
-        default:
-            break;
-    }
-
-    char *str = &wb->buffer[wb->len];
-    char *wstr = str;
-
-    switch (sizeof(void *)) {
-        case 8:
-            do {
-                *wstr++ = (char) ('0' + (uvalue % 10));
-            } while (uvalue /= 10);
-            break;
-
-        case 4:
-            wstr = (uvalue > (unsigned long long) 0xffffffff) ? print_number_llu_r(wstr, uvalue) :
-                   print_number_lu_r(wstr, uvalue);
-            break;
-
-        default:
-            fatal("Netdata supports only 32-bit & 64-bit systems.");
-    }
-
-    // terminate it
-    *wstr = '\0';
-
-    // reverse it
-    char *begin = str, *end = wstr - 1, aux;
-    while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
-
-    size_t len = wstr - str;
-    // return the buffer length
-    wb->len += len;
-}
-
-void buffer_print_ll(BUFFER *wb, long long value)
-{
-    buffer_need_bytes(wb, 50);
-
-    if(value < 0) {
-        buffer_fast_strcat(wb, "-", 1);
-        value = -value;
-    }
-
-    buffer_print_llu(wb, value);
-}
-
-static unsigned char bits03_to_hex[16] = {
-    [0] = '0',
-    [1] = '1',
-    [2] = '2',
-    [3] = '3',
-    [4] = '4',
-    [5] = '5',
-    [6] = '6',
-    [7] = '7',
-    [8] = '8',
-    [9] = '9',
-    [10] = 'A',
-    [11] = 'B',
-    [12] = 'C',
-    [13] = 'D',
-    [14] = 'E',
-    [15] = 'F'
-};
-
-inline void buffer_print_llu_hex(BUFFER *wb, unsigned long long value)
-{
-    unsigned char buffer[sizeof(unsigned long long) * 2 + 2 + 1];   // 8 bytes * 2 + '0x' + '\0'
-    unsigned char *e = &buffer[sizeof(unsigned long long) * 2 + 2];
-    unsigned char *p = e;
-
-    *p-- = '\0';
-    *p-- = bits03_to_hex[value & 0xF];
-    value >>= 4;
-    if(value) {
-        *p-- = bits03_to_hex[value & 0xF];
-        value >>= 4;
-
-        while(value) {
-            *p-- = bits03_to_hex[value & 0xF];
-            value >>= 4;
-
-            if(value) {
-                *p-- = bits03_to_hex[value & 0xF];
-                value >>= 4;
-            }
-        }
-    }
-    *p-- = 'x';
-    *p   = '0';
-
-    buffer_fast_strcat(wb, (char *)p, e - p);
-}
-
-void buffer_print_ll_hex(BUFFER *wb, long long value) {
-    if(value < 0) {
-        buffer_fast_strcat(wb, "-", 1);
-        value = -value;
-    }
-
-    buffer_print_llu_hex(wb, value);
-}
-
-inline void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) {
-    if(unlikely(!txt || !*txt)) return;
-
-    buffer_need_bytes(wb, len + 1);
-
-    char *s = &wb->buffer[wb->len];
-    const char *end = &txt[len + 1];
-
-    while(txt != end)
-        *s++ = *txt++;
-
-    wb->len += len;
-
-    // keep it NULL terminating
-    // not counting it at wb->len
-    wb->buffer[wb->len] = '\0';
-}
-
 void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit) {
     if(unlikely(flags == SN_EMPTY_SLOT)) {
         buffer_fast_strcat(wb, "E", 1);
@@ -268,64 +61,6 @@ void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit) {
         buffer_fast_strcat(wb, "''", 2);
 }
 
-void buffer_strcat(BUFFER *wb, const char *txt)
-{
-    // buffer_sprintf(wb, "%s", txt);
-
-    if(unlikely(!txt || !*txt)) return;
-
-    buffer_need_bytes(wb, 1);
-
-    char *s = &wb->buffer[wb->len], *start, *end = &wb->buffer[wb->size];
-    size_t len = wb->len;
-
-    start = s;
-    while(*txt && s != end)
-        *s++ = *txt++;
-
-    len += s - start;
-
-    wb->len = len;
-    buffer_overflow_check(wb);
-
-    if(unlikely(*txt)) {
-        debug(D_WEB_BUFFER, "strcat(): increasing web_buffer at position %zu, size = %zu\n", wb->len, wb->size);
-        len = strlen(txt);
-        buffer_fast_strcat(wb, txt, len);
-    }
-    else {
-        // terminate the string
-        // without increasing the length
-        buffer_need_bytes(wb, 1);
-        wb->buffer[wb->len] = '\0';
-    }
-}
-
-void buffer_strcat_jsonescape(BUFFER *wb, const char *txt)
-{
-    while(*txt) {
-        switch(*txt) {
-            case '\\':
-                buffer_need_bytes(wb, 2);
-                wb->buffer[wb->len++] = '\\';
-                wb->buffer[wb->len++] = '\\';
-                break;
-            case '"':
-                buffer_need_bytes(wb, 2);
-                wb->buffer[wb->len++] = '\\';
-                wb->buffer[wb->len++] = '"';
-                break;
-            default: {
-                buffer_need_bytes(wb, 1);
-                wb->buffer[wb->len++] = *txt;
-            }
-        }
-        txt++;
-    }
-
-    buffer_overflow_check(wb);
-}
-
 void buffer_strcat_htmlescape(BUFFER *wb, const char *txt)
 {
     while(*txt) {
@@ -412,25 +147,6 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
     // the buffer is \0 terminated by vsnprintf
 }
 
-
-void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value)
-{
-    buffer_need_bytes(wb, 512);
-
-    if(isnan(value) || isinf(value)) {
-        buffer_strcat(wb, "null");
-        return;
-    }
-    else
-        wb->len += print_netdata_double(&wb->buffer[wb->len], value);
-
-    // terminate it
-    buffer_need_bytes(wb, 1);
-    wb->buffer[wb->len] = '\0';
-
-    buffer_overflow_check(wb);
-}
-
 // generate a javascript date, the fastest possible way...
 void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
 {
@@ -536,7 +252,7 @@ BUFFER *buffer_create(size_t size, size_t *statistics)
     b->buffer = mallocz(size + sizeof(BUFFER_OVERFLOW_EOF) + 2);
     b->buffer[0] = '\0';
     b->size = size;
-    b->contenttype = CT_TEXT_PLAIN;
+    b->content_type = CT_TEXT_PLAIN;
     b->statistics = statistics;
     buffer_overflow_init(b);
     buffer_overflow_check(b);
@@ -585,3 +301,200 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
     buffer_overflow_init(b);
     buffer_overflow_check(b);
 }
+
+// ----------------------------------------------------------------------------
+
+void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth, bool add_anonymous_object) {
+    strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
+    strncpyz(wb->json.value_quote,  value_quote, BUFFER_QUOTE_MAX_SIZE);
+
+    wb->json.depth = depth - 1;
+    _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
+
+    if(add_anonymous_object)
+        buffer_fast_strcat(wb, "{", 1);
+}
+
+void buffer_json_finalize(BUFFER *wb) {
+    while(wb->json.depth >= 0) {
+        switch(wb->json.stack[wb->json.depth].type) {
+            case BUFFER_JSON_OBJECT:
+                buffer_json_object_close(wb);
+                break;
+
+            case BUFFER_JSON_ARRAY:
+                buffer_json_array_close(wb);
+                break;
+
+            default:
+                internal_fatal(true, "BUFFER: unknown json member type in stack");
+                break;
+        }
+    }
+    buffer_fast_strcat(wb, "\n", 1);
+}
+
+// ----------------------------------------------------------------------------
+
+const char hex_digits[16] = "0123456789ABCDEF";
+const char base64_digits[64] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+unsigned char hex_value_from_ascii[256];
+unsigned char base64_value_from_ascii[256];
+
+__attribute__((constructor)) void initialize_ascii_maps(void) {
+    for(size_t i = 0 ; i < 256 ; i++) {
+        hex_value_from_ascii[i] = 255;
+        base64_value_from_ascii[i] = 255;
+    }
+
+    for(size_t i = 0; i < 16 ; i++)
+        hex_value_from_ascii[(int)hex_digits[i]] = i;
+
+    for(size_t i = 0; i < 64 ; i++)
+        base64_value_from_ascii[(int)base64_digits[i]] = i;
+}
+
+// ----------------------------------------------------------------------------
+// unit test
+
+static int buffer_expect(BUFFER *wb, const char *expected) {
+    const char *generated = buffer_tostring(wb);
+
+    if(strcmp(generated, expected) != 0) {
+        error("BUFFER: mismatch.\nGenerated:\n%s\nExpected:\n%s\n",
+              generated, expected);
+        return 1;
+    }
+
+    return 0;
+}
+
+static int buffer_uint64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, uint64_t value, const char *expected) {
+    int errors = 0;
+    buffer_flush(wb);
+    buffer_print_uint64_encoded(wb, encoding, value);
+
+    if(expected)
+        errors += buffer_expect(wb, expected);
+
+    uint64_t v = str2ull_encoded(buffer_tostring(wb));
+    if(v != value) {
+        error("BUFFER: string '%s' does resolves to %llu, expected %llu",
+              buffer_tostring(wb), (unsigned long long)v, (unsigned long long)value);
+        errors++;
+    }
+    buffer_flush(wb);
+    return errors;
+}
+
+static int buffer_int64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, int64_t value, const char *expected) {
+    int errors = 0;
+    buffer_flush(wb);
+    buffer_print_int64_encoded(wb, encoding, value);
+
+    if(expected)
+        errors += buffer_expect(wb, expected);
+
+    int64_t v = str2ll_encoded(buffer_tostring(wb));
+    if(v != value) {
+        error("BUFFER: string '%s' does resolves to %lld, expected %lld",
+              buffer_tostring(wb), (long long)v, (long long)value);
+        errors++;
+    }
+    buffer_flush(wb);
+    return errors;
+}
+
+static int buffer_double_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA_DOUBLE value, const char *expected) {
+    int errors = 0;
+    buffer_flush(wb);
+    buffer_print_netdata_double_encoded(wb, encoding, value);
+
+    if(expected)
+        errors += buffer_expect(wb, expected);
+
+    NETDATA_DOUBLE v = str2ndd_encoded(buffer_tostring(wb), NULL);
+    if(v != value) {
+        error("BUFFER: string '%s' does resolves to %.12f, expected %.12f",
+              buffer_tostring(wb), v, value);
+        errors++;
+    }
+    buffer_flush(wb);
+    return errors;
+}
+
+int buffer_unittest(void) {
+    int errors = 0;
+    BUFFER *wb = buffer_create(0, NULL);
+
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "0x0");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "#A");
+
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1676071986, "1676071986");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 1676071986, "0x63E6D432");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 1676071986, "#Bj5tQy");
+
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 18446744073709551615ULL, "18446744073709551615");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 18446744073709551615ULL, "0xFFFFFFFFFFFFFFFF");
+    buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 18446744073709551615ULL, "#P//////////");
+
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "0x0");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "#A");
+
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, -1676071986, "-1676071986");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, -1676071986, "-0x63E6D432");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, -1676071986, "-#Bj5tQy");
+
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, (int64_t)-9223372036854775807ULL, "-9223372036854775807");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, (int64_t)-9223372036854775807ULL, "-0x7FFFFFFFFFFFFFFF");
+    buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, (int64_t)-9223372036854775807ULL, "-#H//////////");
+
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "%0");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "@A");
+
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1.5, "1.5");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 1.5, "%3FF8000000000000");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 1.5, "@D/4AAAAAAAA");
+
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1.23e+14, "123000000000000");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 1.23e+14, "%42DBF78AD3AC0000");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 1.23e+14, "@ELb94rTrAAA");
+
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 9.12345678901234567890123456789e+45, "9.123456789012346128e+45");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 9.12345678901234567890123456789e+45, "%497991C25C9E4309");
+    buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 9.12345678901234567890123456789e+45, "@El5kcJcnkMJ");
+
+    buffer_flush(wb);
+
+    {
+        char buf[1024 + 1];
+        for(size_t i = 0; i < 1024 ;i++)
+            buf[i] = (char)(i % 26) + 'A';
+        buf[1024] = '\0';
+
+        buffer_strcat(wb, buf);
+        errors += buffer_expect(wb, buf);
+    }
+
+    buffer_flush(wb);
+
+    buffer_json_initialize(wb, "\"", "\"", 0, true);
+    buffer_json_finalize(wb);
+    errors += buffer_expect(wb, "{\n}\n");
+
+    buffer_flush(wb);
+
+    buffer_json_initialize(wb, "\"", "\"", 0, true);
+    buffer_json_member_add_string(wb, "hello", "world");
+    buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
+    buffer_json_member_add_object(wb, "object1");
+    buffer_json_member_add_string(wb, "hello", "world");
+    buffer_json_finalize(wb);
+    errors += buffer_expect(wb, "{\n    \"hello\":\"world\",\n    \"alpha\":\"this: \\\" is a double quote\",\n    \"object1\":{\n        \"hello\":\"world\"\n    }\n}\n");
+
+    return errors;
+}
+
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index 0b12c81752..907e586122 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -7,44 +7,74 @@
 
 #define WEB_DATA_LENGTH_INCREASE_STEP 1024
 
+#define BUFFER_JSON_MAX_DEPTH 32
+
+extern const char hex_digits[16];
+extern const char base64_digits[64];
+extern unsigned char hex_value_from_ascii[256];
+extern unsigned char base64_value_from_ascii[256];
+
+typedef enum __attribute__ ((__packed__)) {
+    BUFFER_JSON_EMPTY = 0,
+    BUFFER_JSON_OBJECT,
+    BUFFER_JSON_ARRAY,
+} BUFFER_JSON_NODE_TYPE;
+
+typedef struct web_buffer_json_node {
+    BUFFER_JSON_NODE_TYPE type;
+    uint32_t count:24;
+} BUFFER_JSON_NODE;
+
+#define BUFFER_QUOTE_MAX_SIZE 7
+
+typedef enum __attribute__ ((__packed__)) {
+    WB_CONTENT_CACHEABLE = (1 << 0),
+    WB_CONTENT_NO_CACHEABLE = (1 << 1),
+} BUFFER_OPTIONS;
+
+typedef enum __attribute__ ((__packed__)) {
+    CT_NONE = 0,
+    CT_APPLICATION_JSON,
+    CT_TEXT_PLAIN,
+    CT_TEXT_HTML,
+    CT_APPLICATION_X_JAVASCRIPT,
+    CT_TEXT_CSS,
+    CT_TEXT_XML,
+    CT_APPLICATION_XML,
+    CT_TEXT_XSL,
+    CT_APPLICATION_OCTET_STREAM,
+    CT_APPLICATION_X_FONT_TRUETYPE,
+    CT_APPLICATION_X_FONT_OPENTYPE,
+    CT_APPLICATION_FONT_WOFF,
+    CT_APPLICATION_FONT_WOFF2,
+    CT_APPLICATION_VND_MS_FONTOBJ,
+    CT_IMAGE_SVG_XML,
+    CT_IMAGE_PNG,
+    CT_IMAGE_JPG,
+    CT_IMAGE_GIF,
+    CT_IMAGE_XICON,
+    CT_IMAGE_ICNS,
+    CT_IMAGE_BMP,
+    CT_PROMETHEUS,
+} HTTP_CONTENT_TYPE;
+
 typedef struct web_buffer {
-    size_t size;        	// allocation size of buffer, in bytes
-    size_t len;     		// current data length in buffer, in bytes
-    char *buffer;   		// the buffer itself
-    uint8_t contenttype;	// the content type of the data in the buffer
-    uint8_t options;		// options related to the content
-    time_t date;    		// the timestamp this content has been generated
-    time_t expires;			// the timestamp this content expires
+    size_t size;            // allocation size of buffer, in bytes
+    size_t len;             // current data length in buffer, in bytes
+    char *buffer;           // the buffer itself
+    HTTP_CONTENT_TYPE content_type;    // the content type of the data in the buffer
+    BUFFER_OPTIONS options; // options related to the content
+    time_t date;            // the timestamp this content has been generated
+    time_t expires;         // the timestamp this content expires
     size_t *statistics;
-} BUFFER;
 
-// options
-#define WB_CONTENT_CACHEABLE            1
-#define WB_CONTENT_NO_CACHEABLE         2
-
-// content-types
-#define CT_APPLICATION_JSON             1
-#define CT_TEXT_PLAIN                   2
-#define CT_TEXT_HTML                    3
-#define CT_APPLICATION_X_JAVASCRIPT     4
-#define CT_TEXT_CSS                     5
-#define CT_TEXT_XML                     6
-#define CT_APPLICATION_XML              7
-#define CT_TEXT_XSL                     8
-#define CT_APPLICATION_OCTET_STREAM     9
-#define CT_APPLICATION_X_FONT_TRUETYPE  10
-#define CT_APPLICATION_X_FONT_OPENTYPE  11
-#define CT_APPLICATION_FONT_WOFF        12
-#define CT_APPLICATION_FONT_WOFF2       13
-#define CT_APPLICATION_VND_MS_FONTOBJ   14
-#define CT_IMAGE_SVG_XML                15
-#define CT_IMAGE_PNG                    16
-#define CT_IMAGE_JPG                    17
-#define CT_IMAGE_GIF                    18
-#define CT_IMAGE_XICON                  19
-#define CT_IMAGE_ICNS                   20
-#define CT_IMAGE_BMP                    21
-#define CT_PROMETHEUS                   22
+    struct {
+        char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
+        char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
+        int depth;
+        BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
+    } json;
+} BUFFER;
 
 #define buffer_cacheable(wb)    do { (wb)->options |= WB_CONTENT_CACHEABLE;    if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0)
 #define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE)    (wb)->options &= ~WB_CONTENT_CACHEABLE;  (wb)->expires = 0; } while(0)
@@ -52,12 +82,34 @@ typedef struct web_buffer {
 #define buffer_strlen(wb) ((wb)->len)
 const char *buffer_tostring(BUFFER *wb);
 
-#define buffer_flush(wb) wb->buffer[(wb)->len = 0] = '\0'
-void buffer_reset(BUFFER *wb);
+#define BUFFER_OVERFLOW_EOF "EOF"
+
+#ifdef NETDATA_INTERNAL_CHECKS
+#define buffer_overflow_check(b) _buffer_overflow_check(b)
+#else
+#define buffer_overflow_check(b)
+#endif
+
+static inline void _buffer_overflow_check(BUFFER *b) {
+    assert(b->len <= b->size &&
+                   "BUFFER: length is above buffer size.");
+
+    assert(!(b->buffer && (b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF) != 0)) &&
+                   "BUFFER: detected overflow.");
+}
 
-void buffer_strcat(BUFFER *wb, const char *txt);
-void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len);
-void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value);
+static inline void buffer_flush(BUFFER *wb) {
+    wb->len = 0;
+
+    wb->json.depth = 0;
+    wb->json.stack[0].type = BUFFER_JSON_EMPTY;
+    wb->json.stack[0].count = 0;
+
+    if(wb->buffer)
+        wb->buffer[0] = '\0';
+}
+
+void buffer_reset(BUFFER *wb);
 
 void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
 void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
@@ -69,25 +121,655 @@ void buffer_increase(BUFFER *b, size_t free_size_required);
 void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) PRINTFLIKE(3, 4);
 void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args);
 void buffer_sprintf(BUFFER *wb, const char *fmt, ...) PRINTFLIKE(2,3);
-void buffer_strcat_jsonescape(BUFFER *wb, const char *txt);
 void buffer_strcat_htmlescape(BUFFER *wb, const char *txt);
 
 void buffer_char_replace(BUFFER *wb, char from, char to);
 
 void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit);
 
-char *print_number_lu_r(char *str, unsigned long uvalue);
-char *print_number_llu_r(char *str, unsigned long long uvalue);
-char *print_number_llu_r_smart(char *str, unsigned long long uvalue);
+static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
+    if(unlikely(buffer->len + needed_free_size >= buffer->size))
+        buffer_increase(buffer, needed_free_size + 1);
+}
 
-void buffer_print_llu(BUFFER *wb, unsigned long long uvalue);
-void buffer_print_ll(BUFFER *wb, long long value);
-void buffer_print_llu_hex(BUFFER *wb, unsigned long long value);
-void buffer_print_ll_hex(BUFFER *wb, long long value);
+void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth, bool add_anonymous_object);
+void buffer_json_finalize(BUFFER *wb);
 
-static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
-    if(unlikely(buffer->size - buffer->len < needed_free_size))
-        buffer_increase(buffer, needed_free_size);
+static inline void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) {
+#ifdef NETDATA_INTERNAL_CHECKS
+    assert(wb->json.depth <= BUFFER_JSON_MAX_DEPTH && "BUFFER JSON: max nesting reached");
+#endif
+    wb->json.depth++;
+    wb->json.stack[wb->json.depth].count = 0;
+    wb->json.stack[wb->json.depth].type = type;
+}
+
+static inline void _buffer_json_depth_pop(BUFFER *wb) {
+    wb->json.depth--;
+}
+
+static inline void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) {
+    if(unlikely(!txt || !*txt || !len)) return;
+
+    buffer_need_bytes(wb, len + 1);
+
+    const char *t = txt;
+    const char *e = &txt[len];
+
+    char *d = &wb->buffer[wb->len];
+
+    while(t != e
+#ifdef NETDATA_INTERNAL_CHECKS
+          && *t
+#endif
+            )
+        *d++ = *t++;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+    assert(!(t != e && !*t) && "BUFFER: source string is shorter than the length given.");
+#endif
+
+    wb->len += len;
+    wb->buffer[wb->len] = '\0';
+
+    buffer_overflow_check(wb);
+}
+
+static inline void buffer_strcat(BUFFER *wb, const char *txt) {
+    if(unlikely(!txt || !*txt)) return;
+
+    const char *t = txt;
+    while(*t) {
+        buffer_need_bytes(wb, 100);
+        char *s = &wb->buffer[wb->len];
+        char *d = s;
+        const char *e = &wb->buffer[wb->size];
+
+        while(*t && d < e)
+            *d++ = *t++;
+
+        wb->len += d - s;
+    }
+
+    buffer_need_bytes(wb, 1);
+    wb->buffer[wb->len] = '\0';
+
+    buffer_overflow_check(wb);
+}
+
+static inline void buffer_json_strcat(BUFFER *wb, const char *txt) {
+    if(unlikely(!txt || !*txt)) return;
+
+    const char *t = txt;
+    while(*t) {
+        buffer_need_bytes(wb, 100);
+        char *s = &wb->buffer[wb->len];
+        char *d = s;
+        const char *e = &wb->buffer[wb->size - 1]; // remove 1 to make room for the escape character
+
+        while(*t && d < e) {
+            if(unlikely(*t == '\\' || *t == '\"'))
+                *d++ = '\\';
+
+            *d++ = *t++;
+        }
+
+        wb->len += d - s;
+    }
+
+    buffer_need_bytes(wb, 1);
+    wb->buffer[wb->len] = '\0';
+
+    buffer_overflow_check(wb);
+}
+
+static inline void buffer_json_quoted_strcat(BUFFER *wb, const char *txt) {
+    if(unlikely(!txt || !*txt)) return;
+
+    if(*txt == '"')
+        txt++;
+
+    const char *t = txt;
+    while(*t) {
+        buffer_need_bytes(wb, 100);
+        char *s = &wb->buffer[wb->len];
+        char *d = s;
+        const char *e = &wb->buffer[wb->size - 1]; // remove 1 to make room for the escape character
+
+        while(*t && d < e) {
+            if(unlikely(*t == '"' && !t[1])) {
+                t++;
+                continue;
+            }
+
+            if(unlikely(*t == '\\' || *t == '"'))
+                *d++ = '\\';
+
+            *d++ = *t++;
+        }
+
+        wb->len += d - s;
+    }
+
+    buffer_need_bytes(wb, 1);
+    wb->buffer[wb->len] = '\0';
+
+    buffer_overflow_check(wb);
+}
+
+// This trick seems to give an 80% speed increase in 32bit systems
+// print_number_llu_r() will just print the digits up to the
+// point the remaining value fits in 32 bits, and then calls
+// print_number_lu_r() to print the rest with 32 bit arithmetic.
+
+static inline char *print_uint32_reversed(char *dst, uint32_t value) {
+    char *d = dst;
+    do *d++ = (char)('0' + (value % 10)); while((value /= 10));
+    return d;
+}
+
+static inline char *print_uint64_reversed(char *dst, uint64_t value) {
+#ifdef ENV32BIT
+    if(value <= (uint64_t)0xffffffff)
+        return print_uint32_reversed(dst, value);
+
+    char *d = dst;
+    do *d++ = (char)('0' + (value % 10)); while((value /= 10) && value > (uint64_t)0xffffffff);
+    if(value) return print_uint32_reversed(d, value);
+    return d;
+#else
+    char *d = dst;
+    do *d++ = (char)('0' + (value % 10)); while((value /= 10));
+    return d;
+#endif
+}
+
+static inline char *print_uint32_hex_reversed(char *dst, uint32_t value) {
+    static const char *digits = "0123456789ABCDEF";
+    char *d = dst;
+    do *d++ = digits[value & 0xf]; while((value >>= 4));
+    return d;
+}
+
+static inline char *print_uint64_hex_reversed(char *dst, uint64_t value) {
+#ifdef ENV32BIT
+    if(value <= (uint64_t)0xffffffff)
+        return print_uint32_hex_reversed(dst, value);
+
+    char *d = dst;
+    do *d++ = hex_digits[value & 0xf]; while((value >>= 4) && value > (uint64_t)0xffffffff);
+    if(value) return print_uint32_hex_reversed(d, value);
+    return d;
+#else
+    char *d = dst;
+    do *d++ = hex_digits[value & 0xf]; while((value >>= 4));
+    return d;
+#endif
+}
+
+static inline char *print_uint64_base64_reversed(char *dst, uint64_t value) {
+    char *d = dst