QUERY_TARGET: new query engine for Netdata Agent (#13697)

* initial implementation of QUERY_TARGET

* rrd2rrdr() interface

* rrddim_find_best_tier_for_timeframe() ported

* added dimension filtering

* added db object in query target

* rrd2rrdr() ported

* working on formatters

* working on jsonwrapper

* finally, it compiles...

* 1st run without crashes

* query planer working

* cleanup old code

* review changes

* fix also changing data collection frequency

* fix signess

* fix rrdlabels and dimension ordering

* fixes

* remove unused variable

* ml should accept NULL response from rrd2rrdr()

* number formatting fixes

* more number formatting fixes

* more number formatting fixes

* support mc parallel queries

* formatting and cleanup

* added rrd2rrdr_legacy() as a simplified interface to run a query

* make sure rrdset_find_natural_update_every_for_timeframe() returns a value

* make signed comparisons

* weights endpoint using rrdcontexts

* fix for legacy db modes and cleanup

* fix for chart_ids and remove AR chart from weights endpoint

* Ignore command if not initialized yet

* remove unused members

* properly initialize window

* code cleanup - rrddim linked list is gone; rrdset rwlock is gone too

* reviewed RRDR.internal members

* eliminate unnecessary members of QUERY_TARGET

* more complete query ids; more detailed information on aborted queries

* properly terminate option strings

* query id contains group_options which is controlled by users, so escaping is necessary

* tense in query id

* tense in query id - again

* added the remaining query options to the query id

* Expose hidden option to the dimension

* use the hidden flag when loading context dimensions

* Specify table alias for option

* dont update chart last access time, unless at least a dimension of the chart will be queried

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

21 files changed, 1212 insertions, 1449 deletions

diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 14dad0cb8a..603a171698 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -3,15 +3,14 @@
 #include "libnetdata/libnetdata.h"
 #include "csv.h"
 
-void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) {
-    rrdset_check_rdlock(r->st);
-
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
+    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    RRDDIM *d;
+    const long used = qt->query.used;
 
     // print the csv header
-    for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+    for(c = 0, i = 0; c < used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -23,7 +22,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
         buffer_strcat(wb, separator);
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
-        buffer_strcat(wb, rrddim_name(d));
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
         i++;
     }
@@ -31,7 +30,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
     if(format == DATASOURCE_CSV_MARKDOWN) {
         // print the --- line after header
-        for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0, i = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -89,7 +88,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+            for(c = 0; c < used ;c++) {
                 NETDATA_DOUBLE n = cn[c];
 
                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -103,7 +102,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
 
         // for each dimension
-        for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h
index 99aeb338d7..666d4c6602 100644
--- a/web/api/formatters/csv/csv.h
+++ b/web/api/formatters/csv/csv.h
@@ -5,7 +5,7 @@
 
 #include "web/api/queries/rrdr.h"
 
-void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd);
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines);
 
 #include "../rrd2json.h"
 
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index ae1e55b7a9..608150cba5 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -5,15 +5,7 @@
 #define JSON_DATES_JS 1
 #define JSON_DATES_TIMESTAMP 2
 
-void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct context_param *context_param_list)
-{
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
-
-    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-
-    if (should_lock)
-        rrdset_check_rdlock(r->st);
-
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
     int row_annotations = 0, dates, dates_with_new = 0;
     char kq[2] = "",                        // key quote
@@ -112,21 +104,21 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
     // -------------------------------------------------------------------------
     // print the JSON header
 
+    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    RRDDIM *rd;
+    const long used = qt->query.used;
 
     // print the header lines
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         buffer_fast_strcat(wb, pre_label, pre_label_len);
-        buffer_strcat(wb, rrddim_name(rd));
-//        buffer_strcat(wb, ".");
-//        buffer_strcat(wb, rd->rrdset->name);
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         buffer_fast_strcat(wb, post_label, post_label_len);
         i++;
     }
+
     if(!i) {
         buffer_fast_strcat(wb, pre_label, pre_label_len);
         buffer_fast_strcat(wb, "no data", 7);
@@ -134,7 +126,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
     }
     size_t total_number_of_dimensions = i;
 
-    // print the begin of row data
+    // print the beginning of row data
     buffer_strcat(wb, data_begin);
 
     // if all dimensions are hidden, print a null
@@ -187,7 +179,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
             if(unlikely(row_annotations)) {
                 // google supports one annotation per row
                 int annotation_found = 0;
-                for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) {
+                for(c = 0; c < used ; c++) {
                     if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue;
 
                     if(unlikely(co[c] & RRDR_VALUE_RESET)) {
@@ -222,7 +214,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0; c < used ;c++) {
                 NETDATA_DOUBLE n;
                 if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
                     n = ar[c];
@@ -240,7 +232,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
         }
 
         // for each dimension
-        for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -253,7 +245,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
             buffer_fast_strcat(wb, pre_value, pre_value_len);
 
             if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
-                buffer_sprintf(wb, "%s%s%s: ", kq, rrddim_name(rd), kq);
+                buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq);
 
             if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
                 if(unlikely(options & RRDR_OPTION_NULL2ZERO))
diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h
index 5f5e700820..fb59e5c9a6 100644
--- a/web/api/formatters/json/json.h
+++ b/web/api/formatters/json/json.h
@@ -5,6 +5,6 @@
 
 #include "../rrd2json.h"
 
-void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list);
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable);
 
 #endif //NETDATA_API_FORMATTER_JSON_H
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 811afa8921..5a0abd0d83 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -33,25 +33,17 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL
 }
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data)
+    RRDR_GROUPING group_method)
 {
-    struct context_param *context_param_list = rrdset_query_data->context_param_list;
-    char *chart_label_key = rrdset_query_data->chart_label_key;
-
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
-    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-    uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT));
-
-    if (should_lock)
-        rrdset_check_rdlock(r->st);
+    QUERY_TARGET *qt = r->internal.qt;
 
     long rows = rrdr_rows(r);
     long c, i;
-    RRDDIM *rd;
+    const long query_used = qt->query.used;
 
     //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
     char kq[2] = "",                    // key quote
-            sq[2] = "";                     // string quote
+         sq[2] = "";                    // string quote
 
     if( options & RRDR_OPTION_GOOGLE_JSON ) {
         kq[0] = '\0';
@@ -66,43 +58,43 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %sapi%s: 1,\n"
                        "   %sid%s: %s%s%s,\n"
                        "   %sname%s: %s%s%s,\n"
-                       "   %sview_update_every%s: %d,\n"
-                       "   %supdate_every%s: %d,\n"
-                       "   %sfirst_entry%s: %u,\n"
-                       "   %slast_entry%s: %u,\n"
-                       "   %sbefore%s: %u,\n"
-                       "   %safter%s: %u,\n"
+                       "   %sview_update_every%s: %ld,\n"
+                       "   %supdate_every%s: %ld,\n"
+                       "   %sfirst_entry%s: %ld,\n"
+                       "   %slast_entry%s: %ld,\n"
+                       "   %sbefore%s: %ld,\n"
+                       "   %safter%s: %ld,\n"
                        "   %sgroup%s: %s%s%s,\n"
                        "   %soptions%s: %s"
                    , kq, kq
-                   , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_id(r->st), sq
-                   , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq
+                   , kq, kq, sq, qt->id, sq
+                   , kq, kq, sq, qt->id, sq
                    , kq, kq, r->update_every
-                   , kq, kq, r->st->update_every
-                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st))
-                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st))
-                   , kq, kq, (uint32_t)r->before
-                   , kq, kq, (uint32_t)r->after
+                   , kq, kq, qt->db.minimum_latest_update_every
+                   , kq, kq, qt->db.first_time_t
+                   , kq, kq, qt->db.last_time_t
+                   , kq, kq, r->before
+                   , kq, kq, r->after
                    , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
                    , kq, kq, sq);
 
-    web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options);
+    web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options);
 
     buffer_sprintf(wb, "%s,\n   %sdimension_names%s: [", sq, kq, kq);
 
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         buffer_strcat(wb, sq);
-        buffer_strcat(wb, rrddim_name(rd));
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         buffer_strcat(wb, sq);
         i++;
     }
     if(!i) {
 #ifdef NETDATA_INTERNAL_CHECKS
-        error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", rrdset_id(r->st), r->d, options);
+        error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options);
 #endif
         rows = 0;
         buffer_strcat(wb, sq);
@@ -114,13 +106,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %sdimension_ids%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         buffer_strcat(wb, sq);
-        buffer_strcat(wb, rrddim_id(rd));
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.id));
         buffer_strcat(wb, sq);
         i++;
     }
@@ -132,7 +124,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
     }
     buffer_strcat(wb, "],\n");
 
-    if (rrdset_query_data->show_dimensions) {
+    if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) {
         buffer_sprintf(wb, "   %sfull_dimension_list%s: [", kq, kq);
 
         char name[RRD_ID_LENGTH_MAX * 2 + 2];
@@ -140,35 +132,46 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         struct value_output co = {.c = 0, .wb = wb};
 
-        DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd));
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd));
-            dictionary_set(dict, name, output, len+1);
+        DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->metrics.used ;c++) {
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                      rrdmetric_acquired_id(qt->metrics.array[c]),
+                      rrdmetric_acquired_name(qt->metrics.array[c]));
+
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
+                                rrdmetric_acquired_id(qt->metrics.array[c]),
+                                rrdmetric_acquired_name(qt->metrics.array[c]));
+
+            dictionary_set(dict, name, output, len + 1);
         }
         dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
 
         co.c = 0;
         buffer_sprintf(wb, "],\n   %sfull_chart_list%s: [", kq, kq);
-        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset));
+        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->instances.used ; c++) {
+            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                      rrdinstance_acquired_id(ria),
+                      rrdinstance_acquired_name(ria));
+
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
+                                rrdinstance_acquired_id(ria),
+                                rrdinstance_acquired_name(ria));
+
             dictionary_set(dict, name, output, len + 1);
         }
-
         dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
 
-        RRDSET *st;
         co.c = 0;
         buffer_sprintf(wb, "],\n   %sfull_chart_labels%s: [", kq, kq);
-        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            st = rd->rrdset;
-            if (st->rrdlabels)
-                rrdlabels_walkthrough_read(st->rrdlabels, fill_formatted_callback, dict);
+        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->instances.used ; c++) {
+            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+            rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict);
         }
         dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
@@ -178,8 +181,14 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
     // functions
     {
         DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            chart_functions_to_dict(rd->rrdset, funcs);
+        RRDINSTANCE_ACQUIRED *ria = NULL;
+        for (c = 0; c < query_used ; c++) {
+            QUERY_METRIC *qm = &qt->query.array[c];
+            if(qm->link.ria == ria)
+                continue;
+
+            ria = qm->link.ria;
+            chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs);
         }
 
         buffer_sprintf(wb, "   %sfunctions%s: [", kq, kq);
@@ -194,23 +203,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
         buffer_strcat(wb, "],\n");
     }
 
-    // Composite charts
-    if (context_mode && temp_rd) {
+    // context query
+    if (!qt->request.st) {
         buffer_sprintf(
             wb,
             "   %schart_ids%s: [",
             kq, kq);
 
-        for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) {
+        for (c = 0, i = 0; c < query_used; c++) {
+            QUERY_METRIC *qm = &qt->query.array[c];
+
             if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
                 continue;
+
             if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
                 continue;
 
             if (i)
                 buffer_strcat(wb, ", ");
             buffer_strcat(wb, sq);
-            buffer_strcat(wb, rrdset_id(rd->rrdset));
+            buffer_strcat(wb, string2str(qm->chart.id));
             buffer_strcat(wb, sq);
             i++;
         }
@@ -221,29 +233,29 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             buffer_strcat(wb, sq);
         }
         buffer_strcat(wb, "],\n");
-        if (chart_label_key) {
+        if (qt->instances.chart_label_key_pattern) {
             buffer_sprintf(wb, "   %schart_labels%s: { ", kq, kq);
 
-            SIMPLE_PATTERN *pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-            SIMPLE_PATTERN *original_pattern = pattern;
+            SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern;
             char *label_key = NULL;
             int keys = 0;
             while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
-
                 if (keys)
                     buffer_strcat(wb, ", ");
                 buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq);
                 keys++;
 
-                for (c = 0, i = 0, rd = temp_rd; rd && c < r->d; c++, rd = rd->next) {
+                for (c = 0, i = 0; c < query_used; c++) {
+                    QUERY_METRIC *qm = &qt->query.array[c];
+
                     if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
                         continue;
                     if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
                         continue;
+
                     if (i)
                         buffer_strcat(wb, ", ");
-
-                    rrdlabels_get_value_to_buffer_or_null(rd->rrdset->rrdlabels, wb, label_key, sq, "null");
+                    rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null");
                     i++;
                 }
                 if (!i) {
@@ -255,33 +267,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                 buffer_strcat(wb, "]");
             }
             buffer_strcat(wb, "},\n");
-            simple_pattern_free(original_pattern);
         }
     }
 
     buffer_sprintf(wb, "   %slatest_values%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ;c++) {
+        QUERY_METRIC *qm = &qt->query.array[c];
+
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         i++;
 
-        NETDATA_DOUBLE value = rd->last_stored_value;
+        NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma);
         if (NAN == value)
             buffer_strcat(wb, "null");
         else
             buffer_rrd_value(wb, value);
-        /*
-        storage_number n = rd->values[rrdset_last_slot(r->st)];
-
-        if(!does_storage_number_exist(n))
-            buffer_strcat(wb, "null");
-        else
-            buffer_rrd_value(wb, unpack_storage_number(n));
-        */
     }
     if(!i) {
         rows = 0;
@@ -298,7 +303,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0; c < query_used ;c++) {
                 NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
                 NETDATA_DOUBLE n = cn[c];
 
@@ -311,7 +316,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             if(total == 0) total = 1;
         }
 
-        for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0, i = 0; c < query_used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -361,16 +366,11 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                    , kq, kq
                    );
 
-    for(int tier = 0; tier < storage_tiers ; tier++)
+    for(size_t tier = 0; tier < storage_tiers ; tier++)
         buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]);
 
     buffer_strcat(wb, " ]");
 
-    if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) {
-        buffer_sprintf(wb, ",\n   %schart_variables%s: ", kq, kq);
-        health_api_v1_chart_custom_variables2json(r->st, wb);
-    }
-
     buffer_sprintf(wb, ",\n   %sresult%s: ", kq, kq);
 
     if(string_value) buffer_strcat(wb, sq);
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index e50f39d6d3..91c1475c53 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -8,7 +8,7 @@
 
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    RRDR_GROUPING group_method, QUERY_PARAMS *query_params);
+    RRDR_GROUPING group_method);
 void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index abc350a1aa..ad69a81f6d 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -3,128 +3,6 @@
 #include "web/api/web_api_v1.h"
 #include "database/storage_engine.h"
 
-static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
-{
-    if (unlikely(!temp_rd))
-        return;
-
-    string_freez(temp_rd->id);
-    string_freez(temp_rd->name);
-
-    if (unlikely(archive_mode)) {
-        temp_rd->rrdset->counter--;
-        if (!temp_rd->rrdset->counter) {
-            string_freez(temp_rd->rrdset->id);
-            string_freez(temp_rd->rrdset->name);
-            string_freez(temp_rd->rrdset->context);
-            onewayalloc_freez(owa, temp_rd->rrdset);
-        }
-    }
-
-    for(int tier = 0; tier < storage_tiers ;tier++) {
-        if(!temp_rd->tiers[tier]) continue;
-
-        if(archive_mode) {
-            STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
-            if (eng)
-                eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle);
-        }
-
-        onewayalloc_freez(owa, temp_rd->tiers[tier]);
-    }
-
-    onewayalloc_freez(owa, temp_rd);
-}
-
-static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
-{
-    if (unlikely(!temp_rd))
-        return;
-
-    RRDDIM *t;
-    while (temp_rd) {
-        t = temp_rd->next;
-        free_single_rrdrim(owa, temp_rd, archive_mode);
-        temp_rd = t;
-    }
-}
-
-void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list)
-{
-    if (unlikely(!param_list || !*param_list))
-        return;
-
-    free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
-    onewayalloc_freez(owa, (*param_list));
-    *param_list = NULL;
-}
-
-void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested)
-{
-    RRDDIM *temp_rd = context_param_list->rd;
-    RRDDIM *new_rd_list = NULL, *t;
-    int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE);
-
-    RRDSET *st = temp_rd->rrdset;
-    RRDSET *last_st = st;
-    time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
-    time_t last_last_entry_t = last_entry_t;
-    while (temp_rd) {
-        t = temp_rd->next;
-
-        st = temp_rd->rrdset;
-        if (st == last_st) {
-            last_entry_t = last_last_entry_t;
-        }else {
-            last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
-            last_last_entry_t = last_entry_t;
-            last_st = st;
-        }
-
-        if (last_entry_t >= after_requested) {
-            temp_rd->next = new_rd_list;
-            new_rd_list = temp_rd;
-        } else
-            free_single_rrdrim(owa, temp_rd, is_archived);
-        temp_rd = t;
-    }
-    context_param_list->rd = new_rd_list;
-};
-
-void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st)
-{
-    if (unlikely(!param_list || !st))
-        return;
-
-    if (unlikely(!(*param_list))) {
-        *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
-        (*param_list)->first_entry_t = LONG_MAX;
-        (*param_list)->last_entry_t = 0;
-        (*param_list)->flags = CONTEXT_FLAGS_CONTEXT;
-        (*param_list)->rd = NULL;
-    }
-
-    st->last_accessed_time = now_realtime_sec();
-    (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st));
-    (*param_list)->last_entry_t  = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st));
-
-    RRDDIM *rd1;
-    rrddim_forea