JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)

* first work on standardizing json formatting

* renamed old grouping to time_grouping and added group_by

* add dummy functions to enable compilation

* buffer json api work

* jsonwrap opening with buffer_json_X() functions

* cleanup

* storage for quotes

* optimize buffer printing for both numbers and strings

* removed ; from define

* contexts json generation using the new json functions

* fix buffer overflow at unit test

* weights endpoint using new json api

* fixes to weights endpoint

* check buffer overflow on all buffer functions

* do synchronous queries for weights

* buffer_flush() now resets json state too

* content type typedef

* print double values that are above the max 64-bit value

* str2ndd() can now parse values above UINT64_MAX

* faster number parsing by avoiding double calculations as much as possible

* faster number parsing

* faster hex parsing

* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers

* full printing and parsing without using library functions - and related unit tests

* added IEEE754 streaming capability to enable streaming of double values in hex

* streaming and replication to transfer all values in hex

* use our own str2ndd for set2

* remove subnormal check from ieee

* base64 encoding for numbers, instead of hex

* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision

* str2ndd_encoded() parses all encoding formats, including integers

* prevent uninitialized use

* /api/v1/info using the new json API

* Fix error when compiling with --disable-ml

* Remove redundant 'buffer_unittest' declaration

* Fix formatting

* Fix formatting

* Fix formatting

* fix buffer unit test

* apps.plugin using the new JSON API

* make sure the metrics registry does not accept negative timestamps

* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache

* Fix more formatting

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

23 files changed, 729 insertions, 916 deletions

diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index ca0f4b7a03..5a2a16c657 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -767,7 +767,7 @@ void buffer_svg(BUFFER *wb, const char *label,
     label_color_parsed = parse_color_argument(label_color, "555");
     value_color_parsed = parse_color_argument(value_color_buffer, "555");
 
-    wb->contenttype = CT_IMAGE_SVG_XML;
+    wb->content_type = CT_IMAGE_SVG_XML;
 
     total_width  = total_width * scale / 100.0;
     height       = height      * scale / 100.0;
@@ -923,7 +923,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
         else if(!strcmp(name, "points")) points_str = value;
         else if(!strcmp(name, "group_options")) group_options = value;
         else if(!strcmp(name, "group")) {
-            group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);
+            group = time_grouping_parse(value, RRDR_GROUPING_AVERAGE);
         }
         else if(!strcmp(name, "options")) {
             options |= web_client_api_request_v1_data_options(value);
diff --git a/web/api/exporters/allmetrics.c b/web/api/exporters/allmetrics.c
index 88065400d6..43a3dd78f3 100644
--- a/web/api/exporters/allmetrics.c
+++ b/web/api/exporters/allmetrics.c
@@ -90,17 +90,17 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
 
     switch(format) {
         case ALLMETRICS_JSON:
-            w->response.data->contenttype = CT_APPLICATION_JSON;
+            w->response.data->content_type = CT_APPLICATION_JSON;
             rrd_stats_api_v1_charts_allmetrics_json(host, filter, w->response.data);
             return HTTP_RESP_OK;
 
         case ALLMETRICS_SHELL:
-            w->response.data->contenttype = CT_TEXT_PLAIN;
+            w->response.data->content_type = CT_TEXT_PLAIN;
             rrd_stats_api_v1_charts_allmetrics_shell(host, filter, w->response.data);
             return HTTP_RESP_OK;
 
         case ALLMETRICS_PROMETHEUS:
-            w->response.data->contenttype = CT_PROMETHEUS;
+            w->response.data->content_type = CT_PROMETHEUS;
             rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
                     host
                     , filter
@@ -113,7 +113,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
             return HTTP_RESP_OK;
 
         case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
-            w->response.data->contenttype = CT_PROMETHEUS;
+            w->response.data->content_type = CT_PROMETHEUS;
             rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
                     host
                     , filter
@@ -126,7 +126,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
             return HTTP_RESP_OK;
 
         default:
-            w->response.data->contenttype = CT_TEXT_PLAIN;
+            w->response.data->content_type = CT_TEXT_PLAIN;
             buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "', '" ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported.");
             return HTTP_RESP_BAD_REQUEST;
     }
diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c
index 61a9ecf2f2..4b6b095c2a 100644
--- a/web/api/formatters/charts2json.c
+++ b/web/api/formatters/charts2json.c
@@ -137,55 +137,3 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
 
     buffer_sprintf(wb, "\n\t]\n}\n");
 }
-
-// generate collectors list for the api/v1/info call
-
-struct collector {
-    const char *plugin;
-    const char *module;
-};
-
-struct array_printer {
-    int c;
-    BUFFER *wb;
-};
-
-static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {
-    struct array_printer *ap = (struct array_printer *)data;
-    BUFFER *wb = ap->wb;
-    struct collector *col=(struct collector *) entry;
-    if(ap->c) buffer_strcat(wb, ",");
-    buffer_strcat(wb, "\n\t\t{\n\t\t\t\"plugin\": \"");
-    buffer_strcat(wb, col->plugin);
-    buffer_strcat(wb, "\",\n\t\t\t\"module\": \"");
-    buffer_strcat(wb, col->module);
-    buffer_strcat(wb, "\"\n\t\t}");
-    (ap->c)++;
-    return 0;
-}
-
-void chartcollectors2json(RRDHOST *host, BUFFER *wb) {
-    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
-    RRDSET *st;
-    char name[500];
-
-    time_t now = now_realtime_sec();
-    rrdset_foreach_read(st, host) {
-        if (rrdset_is_available_for_viewers(st)) {
-            struct collector col = {
-                    .plugin = rrdset_plugin_name(st),
-                    .module = rrdset_module_name(st)
-            };
-            sprintf(name, "%s:%s", col.plugin, col.module);
-            dictionary_set(dict, name, &col, sizeof(struct collector));
-            st->last_accessed_time_s = now;
-        }
-    }
-    rrdset_foreach_done(st);
-    struct array_printer ap = {
-            .c = 0,
-            .wb = wb
-    };
-    dictionary_walkthrough_read(dict, print_collector_callback, &ap);
-    dictionary_destroy(dict);
-}
diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h
index d4b04af585..96720d4b4c 100644
--- a/web/api/formatters/charts2json.h
+++ b/web/api/formatters/charts2json.h
@@ -6,7 +6,6 @@
 #include "rrd2json.h"
 
 void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
-void chartcollectors2json(RRDHOST *host, BUFFER *wb);
 const char* get_release_channel();
 
 #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 18009f1461..1713c84ad9 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -11,9 +11,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
     // print the csv header
     for(c = 0, i = 0; c < used ; c++) {
-        if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-        if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-        if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
+        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
 
         if(!i) {
             buffer_strcat(wb, startline);
@@ -32,9 +31,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
     if(format == DATASOURCE_CSV_MARKDOWN) {
         // print the --- line after header
         for(c = 0, i = 0; c < used ;c++) {
-            if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-            if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-            if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
+            if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
             if(!i) {
                 buffer_strcat(wb, startline);
@@ -76,7 +74,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
         if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
             // print the timestamp of the line
-            buffer_rrd_value(wb, (NETDATA_DOUBLE)now);
+            buffer_print_netdata_double(wb, (NETDATA_DOUBLE) now);
             // in ms
             if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
         }
@@ -107,9 +105,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
         // for each dimension
         for(c = 0; c < used ;c++) {
-            if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-            if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-            if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
+            if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
             buffer_strcat(wb, separator);
 
@@ -137,7 +134,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
                     if(n > r->max) r->max = n;
                 }
 
-                buffer_rrd_value(wb, n);
+                buffer_print_netdata_double(wb, n);
             }
         }
 
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 3cad3e914a..d1e71851ce 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -42,7 +42,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
         strcpy(post_value,         "}");
         strcpy(post_line,          "]}");
         snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n [\n", kq, kq);
-        strcpy(finish,             "\n]\n}");
+        strcpy(finish,             "\n  ]\n }");
 
         snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
         snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
@@ -69,9 +69,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             dates_with_new = 0;
         }
         if( options & RRDR_OPTION_OBJECTSROWS )
-            strcpy(pre_date, " { ");
+            strcpy(pre_date, "   {");
         else
-            strcpy(pre_date, " [ ");
+            strcpy(pre_date, "   [");
         strcpy(pre_label,  ",\"");
         strcpy(post_label, "\"");
         strcpy(pre_value,  ",");
@@ -79,10 +79,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             strcpy(post_line, "}");
         else
             strcpy(post_line, "]");
-        snprintfz(data_begin, 100, "],\n    %sdata%s:\n [\n", kq, kq);
-        strcpy(finish,             "\n]\n}");
+        snprintfz(data_begin, 100, "],\n  %sdata%s:[\n", kq, kq);
+        strcpy(finish,             "\n  ]\n }");
 
-        buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
+        buffer_sprintf(wb, "{\n  %slabels%s:[", kq, kq);
         buffer_sprintf(wb, "%stime%s", sq, sq);
 
         if( options & RRDR_OPTION_OBJECTSROWS )
@@ -110,9 +110,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
 
     // print the header lines
     for(c = 0, i = 0; c < used ; c++) {
-        if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-        if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-        if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
+        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
 
         buffer_fast_strcat(wb, pre_label, pre_label_len);
         buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
@@ -203,7 +202,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
                 buffer_fast_strcat(wb, object_rows_time, object_rows_time_len);
 
-            buffer_rrd_value(wb, (NETDATA_DOUBLE)r->t[i]);
+            buffer_print_netdata_double(wb, (NETDATA_DOUBLE) r->t[i]);
 
             // in ms
             if(unlikely(options & RRDR_OPTION_MILLISECONDS))
@@ -236,9 +235,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
 
         // for each dimension
         for(c = 0; c < used ;c++) {
-            if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-            if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-            if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
+            if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
             NETDATA_DOUBLE n;
             if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
@@ -273,7 +271,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
                     if(n > r->max) r->max = n;
                 }
 
-                buffer_rrd_value(wb, n);
+                buffer_print_netdata_double(wb, n);
             }
 
             buffer_fast_strcat(wb, post_value, post_value_len);
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index aa663495a7..ba71ccc86d 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -2,433 +2,388 @@
 
 #include "json_wrapper.h"
 
-struct value_output {
-    int c;
-    BUFFER *wb;
-};
-
-static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {
-    struct value_output *ap = (struct value_output *)data;
-    BUFFER *wb = ap->wb;
-    char *output = (char *) entry;
-    if(ap->c) buffer_strcat(wb, ",");
-    buffer_strcat(wb, output);
-    (ap->c)++;
-    return 0;
-}
-
-static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
-    (void)ls;
-    DICTIONARY *dict = (DICTIONARY *)data;
-    char n[RRD_ID_LENGTH_MAX * 2 + 2];
-    char output[RRD_ID_LENGTH_MAX * 2 + 8];
-    char v[RRD_ID_LENGTH_MAX * 2 + 1];
-
-    sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2);
-    int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v);
-    snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v);
-    dictionary_set(dict, n, output, len + 1);
-
-    return 1;
-}
-
-void rrdr_show_plan(RRDR *r, BUFFER *wb, const char *kq, const char *sq __maybe_unused) {
+void jsonwrap_query_plan(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
 
-    buffer_sprintf(wb, "\n\t%squery_plan%s: {", kq, kq);
-
+    buffer_json_member_add_object(wb, "query_plan");
     for(size_t m = 0; m < qt->query.used; m++) {
         QUERY_METRIC *qm = &qt->query.array[m];
 
-        if(m)
-            buffer_strcat(wb, ",");
-
-        buffer_sprintf(wb, "\n\t\t%s%s%s: {", kq, string2str(qm->dimension.id), kq);
-
-        buffer_sprintf(wb, "\n\t\t\t%splans%s: [", kq, kq);
-        for(size_t p = 0; p < qm->plan.used ;p++) {
-            QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
-            if(p)
-                buffer_strcat(wb, ",");
-
-            buffer_strcat(wb, "\n\t\t\t\t{");
-            buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, qp->tier);
-            buffer_sprintf(wb, "\n\t\t\t\t\t%safter%s: %ld,", kq, kq, qp->after);
-            buffer_sprintf(wb, "\n\t\t\t\t\t%sbefore%s: %ld", kq, kq, qp->before);
-            buffer_strcat(wb, "\n\t\t\t\t}");
-        }
-        buffer_strcat(wb, "\n\t\t\t],");
-
-        buffer_sprintf(wb, "\n\t\t\t%stiers%s: [", kq, kq);
-        for(size_t tier = 0; tier < storage_tiers ;tier++) {
-            if(tier)
-                buffer_strcat(wb, ",");
-
-            buffer_strcat(wb, "\n\t\t\t\t{");
-            buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, tier);
-            buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_first_time%s: %ld,", kq, kq, qm->tiers[tier].db_first_time_s);
-            buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_last_time%s: %ld,", kq, kq, qm->tiers[tier].db_last_time_s);
-            buffer_sprintf(wb, "\n\t\t\t\t\t%sweight%s: %ld", kq, kq, qm->tiers[tier].weight);
-            buffer_strcat(wb, "\n\t\t\t\t}");
+        buffer_json_member_add_object(wb, string2str(qm->dimension.id));
+        {
+            buffer_json_member_add_array(wb, "plans");
+            for (size_t p = 0; p < qm->plan.used; p++) {
+                QUERY_PLAN_ENTRY *qp = &qm->plan.array[p];
+
+                buffer_json_add_array_item_object(wb);
+                buffer_json_member_add_uint64(wb, "tier", qp->tier);
+                buffer_json_member_add_time_t(wb, "after", qp->after);
+                buffer_json_member_add_time_t(wb, "before", qp->before);
+                buffer_json_object_close(wb);
+            }
+            buffer_json_array_close(wb);
+
+            buffer_json_member_add_array(wb, "tiers");
+            for (size_t tier = 0; tier < storage_tiers; tier++) {
+                buffer_json_add_array_item_object(wb);
+                buffer_json_member_add_uint64(wb, "tier", tier);
+                buffer_json_member_add_time_t(wb, "db_first_time", qm->tiers[tier].db_first_time_s);
+                buffer_json_member_add_time_t(wb, "db_last_time", qm->tiers[tier].db_last_time_s);
+                buffer_json_member_add_int64(wb, "weight", qm->tiers[tier].weight);
+                buffer_json_object_close(wb);
+            }
+            buffer_json_array_close(wb);
         }
-        buffer_strcat(wb, "\n\t\t\t]");
-
-        buffer_strcat(wb, "\n\t\t}");
+        buffer_json_object_close(wb);
     }
-
-    buffer_strcat(wb, "\n\t},");
+    buffer_json_object_close(wb);
 }
 
-void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    RRDR_GROUPING group_method)
-{
+static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
     QUERY_TARGET *qt = r->internal.qt;
-
-    long rows = rrdr_rows(r);
-    long c, i;
     const long query_used = qt->query.used;
+    long c, i;
 
-    //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
-    char kq[2] = "",                    // key quote
-         sq[2] = "";                    // string quote
-
-    if( options & RRDR_OPTION_GOOGLE_JSON ) {
-        kq[0] = '\0';
-        sq[0] = '\'';
-    }
-    else {
-        kq[0] = '"';
-        sq[0] = '"';
-    }
-
-    buffer_sprintf(wb, "{\n"
-                       "   %sapi%s: 1,\n"
-                       "   %sid%s: %s%s%s,\n"
-                       "   %sname%s: %s%s%s,\n"
-                       "   %sview_update_every%s: %lld,\n"
-                       "   %supdate_every%s: %lld,\n"
-                       "   %sfirst_entry%s: %lld,\n"
-                       "   %slast_entry%s: %lld,\n"
-                       "   %sbefore%s: %lld,\n"
-                       "   %safter%s: %lld,\n"
-                       "   %sgroup%s: %s%s%s,\n"
-                       "   %soptions%s: %s"
-                   , kq, kq
-                   , kq, kq, sq, qt->id, sq
-                   , kq, kq, sq, qt->id, sq
-                   , kq, kq, (long long)r->update_every
-                   , kq, kq, (long long)qt->db.minimum_latest_update_every_s
-                   , kq, kq, (long long)qt->db.first_time_s
-                   , kq, kq, (long long)qt->db.last_time_s
-                   , kq, kq, (long long)r->before
-                   , kq, kq, (long long)r->after
-                   , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
-                   , kq, kq, sq);
-
-    web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options);
-
-    buffer_sprintf(wb, "%s,\n   %sdimension_names%s: [", sq, kq, kq);
-
+    buffer_json_member_add_array(wb, key);
     for(c = 0, i = 0; c < query_used ; c++) {
-        if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-        if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-        if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
-
-        if(i) buffer_strcat(wb, ", ");
-        buffer_strcat(wb, sq);
-        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
-        buffer_strcat(wb, sq);
+        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
+
+        buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.name));
         i++;
     }
-    if(!i) {
-#ifdef NETDATA_INTERNAL_CHECKS
-        error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options);
-#endif
-        rows = 0;
-        buffer_strcat(wb, sq);
-        buffer_strcat(wb, "no data");
-        buffer_strcat(wb, sq);
-    }
+    buffer_json_array_close(wb);
 
-    buffer_sprintf(wb, "],\n"
-                       "   %sdimension_ids%s: ["
-                   , kq, kq);
+    return i;
+}
+
+static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+    QUERY_TARGET *qt = r->internal.qt;
+    const long query_used = qt->query.used;
+    long c, i;
 
+    buffer_json_member_add_array(wb, key);
     for(c = 0, i = 0; c < query_used ; c++) {
-        if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
-        if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
-        if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
-
-        if(i) buffer_strcat(wb, ", ");
-        buffer_strcat(wb, sq);
-        buffer_strcat(wb, string2str(qt->query.array[c].dimension.id));
-        buffer_strcat(wb, sq);
+        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
+
+        buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.id));
         i++;
     }
-    if(!i) {
-        rows = 0;
-        buffer_strcat(wb, sq);
-        buffer_strcat(wb, "no data");
-        buffer_strcat(wb, sq);
-    }
-    buffer_strcat(wb, "],\n");
+    buffer_json_array_close(wb);
 
-    if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) {
-        buffer_sprintf(wb, "   %sfull_dimension_list%s: [", kq, kq);
+    return i;
+}
 
-        char name[RRD_ID_LENGTH_MAX * 2 + 2];
-        char output[RRD_ID_LENGTH_MAX * 2 + 8];
+static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
+    QUERY_TARGET *qt = r->internal.qt;
+    const long query_used = qt->query.used;
+    long c, i;
 
-        struct value_output co = {.c = 0, .wb = wb};
+    buffer_json_member_add_array(wb, key);
+    for (c = 0, i = 0; c < query_used; c++) {
+        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
 
-        DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-        for (c = 0; c < (long)qt->metrics.used ;c++) {
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
-                      rrdmetric_acquired_id(qt->metrics.array[c]),
-                      rrdmetric_acquired_name(qt->metrics.array[c]));
+        QUERY_METRIC *qm = &qt->query.array[c];
+        buffer_json_add_array_item_string(wb, string2str(qm->chart.id));
+        i++;
+    }
+    buffer_json_array_close(wb);
 
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
-                                rrdmetric_acquired_id(qt->metrics.array[c]),
-                                rrdmetric_acquired_name(qt->metrics.array[c]));
+    return i;
+}
 
-            dictionary_set(dict, name, output, len + 1);
-        }
-        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
-        dictionary_destroy(dict);
+struct rrdlabels_formatting_v2 {
+    BUFFER *wb;
+    DICTIONARY *dict;
+};
 
-        co.c = 0;
-        buffer_sprintf(wb, "],\n   %sfull_chart_list%s: [", kq, kq);
-        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-        for (c = 0; c < (long)qt->instances.used ; c++) {
-            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
+    struct rrdlabels_formatting_v2 *t = data;
+    BUFFER *wb = t->wb;
+    DICTIONARY *dict = t->dict;
 
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
-                      rrdinstance_acquired_id(ria),
-                      rrdinstance_acquired_name(ria));
+    char n[RRD_ID_LENGTH_MAX * 2 + 2];
+    snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
+
+    bool existing = 0;
+    bool *set = dictionary_set(dict, n, &existing, sizeof(bool));
+    if(!*set) {
+        *set = true;
+        buffer_json_add_array_item_array(wb);
+        buffer_json_add_array_item_string(wb, name);
+        buffer_json_add_array_item_string(wb, value);
+        buffer_json_array_close(wb);
+    }
+
+    return 1;
+}
 
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
-                                rrdinstance_acquired_id(ria),
-                                rrdinstance_acquired_name(ria));
+static inline void jsonwrap_full_dimension_list(BUFFER *wb, RRDR *r) {
+    QUERY_TARGET *qt = r->internal.qt;
 
-            dictionary_set(dict, name, output, len + 1);
+    char name[RRD_ID_LENGTH_MAX * 2 + 2];
+
+    buffer_json_member_add_array(wb, "full_dimension_list");
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+    for (long c = 0; c < (long)qt->metrics.used ;c++) {
+        RRDMETRIC_ACQUIRED *rma = qt->metrics.array[c];
+
+        snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                  rrdmetric_acquired_id(rma),
+                  rrdmetric_acquired_name(rma));
+
+        bool existing = 0;
+        bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
+        if(!*set) {
+            *set = true;
+            buffer_json_add_array_item_array(wb);
+            buffer_json_add_array_item_string(wb, rrdmetric_acquired_id(rma));
+            buffer_json_add_array_item_string(wb, rrdmetric_acquired_name(rma));
+            buffer_json_array_close(wb);
         }
-        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
-        dictionary_destroy(dict);
-
-        co.c = 0;
-        buffer_sprintf(wb, "],\n   %sfull_chart_labels%s: [", kq, kq);
-        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-        for (c = 0; c < (long)qt->instances.used ; c++) {
-            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
-            rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict);
+    }
+    dictionary_destroy(dict);
+    buffer_json_array_close(wb);
+
+    buffer_json_member_add_array(wb, "full_chart_list");
+    dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+    for (long c = 0; c < (long)qt->instances.used ; c++) {
+        RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+
+        snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                  rrdinstance_acquired_id(ria),
+                  rrdinstance_acquired_name(ria));
+
+        bool existing = 0;
+        bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
+        if(!*set) {
+            *set = true;
+            buffer_json_add_array_item_array(wb);
+            buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(ria));
+            buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(ria));
+            buffer_json_array_close(wb);
         }
-        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
-        dictionary_destroy(dict);
-        buffer_strcat(wb, "],\n");
     }
+    dictionary_destroy(dict);
+    buffer_json_array_close(wb);
+
+    buffer_json_member_add_array(wb, "full_chart_labels");
+    struct rrdlabels_formatting_v2 t = {
+            .wb = wb,
+            .dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE),
+    };
+    for (long c = 0; c < (long)qt->instances.used ; c++) {
+        RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+        rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t);
+    }
+    dictionary_destroy(t.dict);
+    buffer_json_array_close(wb);
+}
 
-    // functions
-    {
-        DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-        RRDINSTANCE_ACQUIRED *ria = NULL;</