Added context parameter to the data endpoint (#9931)

Added functionality to support composite charts

16 files changed, 244 insertions, 91 deletions

diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index 116fe7061d..10fa8e4ba7 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -1673,7 +1673,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
     update_every = REGION_UPDATE_EVERY[current_region];
     long points = (time_end - time_start) / update_every - 1;
     for (i = 0 ; i < CHARTS ; ++i) {
-        RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL);
+        RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL);
         if (!r) {
             fprintf(stderr, "    DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
             return ++errors;
@@ -1792,7 +1792,7 @@ int test_dbengine(void)
     long points = (time_end[REGIONS - 1] - time_start[0]) / update_every - 1; // cover all time regions with RRDR
     long point_offset = (time_start[current_region] - time_start[0]) / update_every;
     for (i = 0 ; i < CHARTS ; ++i) {
-        RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL);
+        RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL);
         if (!r) {
             fprintf(stderr, "    DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
             ++errors;
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index 8a78e4bb9a..a8649a2238 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -376,7 +376,7 @@ static inline uint32_t *pginfo_to_points(struct rrdeng_page_info *page_info)
  * @return number of regions with different data collection intervals.
  */
 unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
-                                         struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp)
+                                         struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd)
 {
     struct pg_cache_page_index *page_index;
     struct rrdengine_instance *ctx;
@@ -397,7 +397,7 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e
     page_info_array = NULL;
 
     rrdset_rdlock(st);
-    for(rd_iter = st->dimensions, rd = NULL, min_time = (usec_t)-1 ; rd_iter ; rd_iter = rd_iter->next) {
+    for(rd_iter = temp_rd?temp_rd:st->dimensions, rd = NULL, min_time = (usec_t)-1 ; rd_iter ; rd_iter = rd_iter->next) {
         /*
          * Choose oldest dimension as reference. This is not equivalent to the union of all dimensions
          * but it is a best effort approximation with a bias towards older metrics in a chart. It
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index 8809447d80..8d4398c56b 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -43,7 +43,7 @@ extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_n
 extern int rrdeng_store_metric_finalize(RRDDIM *rd);
 extern unsigned
     rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
-                                    struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp);
+                                    struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd);
 extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
                                     time_t start_time, time_t end_time);
 extern storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *current_time);
diff --git a/database/rrd.h b/database/rrd.h
index 039a489672..0825c955f8 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -1053,6 +1053,19 @@ static inline time_t rrdset_first_entry_t(RRDSET *st) {
     }
 }
 
+// get the timestamp of the last entry in the round robin database
+static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
+    if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+        return rd->state->query_ops.latest_time(rd);
+    return (time_t)rd->rrdset->last_updated.tv_sec;
+}
+
+static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
+    if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+        return rd->state->query_ops.oldest_time(rd);
+    return (time_t)(rd->rrdset->last_updated.tv_sec - rrdset_duration(rd->rrdset));
+}
+
 time_t rrdhost_last_entry_t(RRDHOST *h);
 
 // get the last slot updated in the round robin database
diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 53bf29828e..da0a6b5837 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -3,7 +3,7 @@
 #include "libnetdata/libnetdata.h"
 #include "csv.h"
 
-void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) {
     rrdset_check_rdlock(r->st);
 
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
@@ -11,7 +11,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
     RRDDIM *d;
 
     // print the csv header
-    for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+    for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -31,7 +31,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
     if(format == DATASOURCE_CSV_MARKDOWN) {
         // print the --- line after header
-        for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -89,7 +89,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+            for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
                 calculated_number n = cn[c];
 
                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -103,7 +103,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
 
         // for each dimension
-        for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h
index a89742d346..cf6020de4d 100644
--- a/web/api/formatters/csv/csv.h
+++ b/web/api/formatters/csv/csv.h
@@ -5,7 +5,7 @@
 
 #include "web/api/queries/rrdr.h"
 
-extern void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines);
+extern void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd);
 
 #include "../rrd2json.h"
 
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 66b3b9c837..f28eb5738b 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -5,7 +5,7 @@
 #define JSON_DATES_JS 1
 #define JSON_DATES_TIMESTAMP 2
 
-void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd) {
     rrdset_check_rdlock(r->st);
 
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
@@ -94,12 +94,14 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
     RRDDIM *rd;
 
     // print the header lines
-    for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         buffer_strcat(wb, pre_label);
         buffer_strcat(wb, rd->name);
+//        buffer_strcat(wb, ".");
+//        buffer_strcat(wb, rd->rrdset->name);
         buffer_strcat(wb, post_label);
         i++;
     }
@@ -154,7 +156,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
             if(row_annotations) {
                 // google supports one annotation per row
                 int annotation_found = 0;
-                for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
+                for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) {
                     if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue;
 
                     if(co[c] & RRDR_VALUE_RESET) {
@@ -185,7 +187,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
                 calculated_number n = cn[c];
 
                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -199,7 +201,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
         }
 
         // for each dimension
-        for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h
index 01363ce014..6d73a3ffe4 100644
--- a/web/api/formatters/json/json.h
+++ b/web/api/formatters/json/json.h
@@ -5,6 +5,6 @@
 
 #include "../rrd2json.h"
 
-extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable);
+extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd);
 
 #endif //NETDATA_API_FORMATTER_JSON_H
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 5ef2caf101..e838a969f8 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -2,7 +2,7 @@
 
 #include "json_wrapper.h"
 
-void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value) {
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd) {
     rrdset_check_rdlock(r->st);
 
     long rows = rrdr_rows(r);
@@ -34,8 +34,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %safter%s: %u,\n"
                        "   %sdimension_names%s: ["
                    , kq, kq
-                   , kq, kq, sq, r->st->id, sq
-                   , kq, kq, sq, r->st->name, sq
+                   , kq, kq, sq, temp_rd?r->st->context:r->st->id, sq
+                   , kq, kq, sq, temp_rd?r->st->context:r->st->name, sq
                    , kq, kq, r->update_every
                    , kq, kq, r->st->update_every
                    , kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
@@ -44,7 +44,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                    , kq, kq, (uint32_t)r->after
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -68,7 +68,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %sdimension_ids%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -85,11 +85,41 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
         buffer_strcat(wb, sq);
     }
 
+    // Composite charts
+    if (temp_rd) {
+        buffer_sprintf(
+            wb,
+            "],\n"
+            "   %schart_ids%s: [",
+            kq, kq);
+
+        for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) {
+            if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
+                continue;
+            if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
+                continue;
+
+            if (i)
+                buffer_strcat(wb, ", ");
+            buffer_strcat(wb, sq);
+            buffer_strcat(wb, rd->rrdset->name);
+            buffer_strcat(wb, sq);
+            i++;
+        }
+        if (!i) {
+            rows = 0;
+            buffer_strcat(wb, sq);
+            buffer_strcat(wb, "no data");
+            buffer_strcat(wb, sq);
+        }
+    }
+
+
     buffer_sprintf(wb, "],\n"
                        "   %slatest_values%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -125,7 +155,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
                 calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
                 calculated_number n = cn[c];
 
@@ -138,7 +168,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             if(total == 0) total = 1;
         }
 
-        for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index 7cb7d34530..b502f02862 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -5,7 +5,7 @@
 
 #include "rrd2json.h"
 
-extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value);
+extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd);
 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 5672c69521..7b07d398ed 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -2,6 +2,26 @@
 
 #include "web/api/web_api_v1.h"
 
+static inline void free_temp_rrddim(RRDDIM *temp_rd)
+{
+    if (unlikely(!temp_rd))
+        return;
+
+    RRDDIM *t;
+    while (temp_rd) {
+        t = temp_rd->next;
+        freez((char *)temp_rd->id);
+        freez((char *)temp_rd->name);
+#ifdef ENABLE_DBENGINE
+        if (temp_rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            freez(temp_rd->state->metric_uuid);
+#endif
+        freez(temp_rd->state);
+        freez(temp_rd);
+        temp_rd = t;
+    }
+}
+
 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
     rrdset2json(st, wb, NULL, NULL, 0);
 }
@@ -69,7 +89,10 @@ int rrdset2value_api_v1(
         , time_t *db_before
         , int *value_is_null
 ) {
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions);
+    RRDDIM *temp_rd = NULL;
+
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, temp_rd);
+
     if(!r) {
         if(value_is_null) *value_is_null = 1;
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -98,6 +121,8 @@ int rrdset2value_api_v1(
     long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
     *n = rrdr2value(r, i, options, value_is_null);
 
+    free_temp_rrddim(temp_rd);
+
     rrdr_free(r);
     return HTTP_RESP_OK;
 }
@@ -114,10 +139,45 @@ int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
+        , char *context
 ) {
     st->last_accessed_time = now_realtime_sec();
 
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL);
+    RRDDIM *temp_rd = NULL;
+
+    if (context) {
+        rrdhost_rdlock(st->rrdhost);
+        RRDSET *st1;
+        rrdset_foreach_read(st1, st->rrdhost) {
+            if (strcmp(st1->context, context) == 0) {
+                // Loop the dimensions of the chart
+                RRDDIM  *rd1;
+                rrdset_rdlock(st1);
+                rrddim_foreach_read(rd1, st1) {
+                    RRDDIM *rd = mallocz(rd1->memsize);
+                    memcpy(rd, rd1, rd1->memsize);
+                    rd->id = strdupz(rd1->id);
+                    rd->name = strdupz(rd1->name);
+                    rd->state = mallocz(sizeof(*rd->state));
+                    memcpy(rd->state, rd1->state, sizeof(*rd->state));
+                    memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
+                    memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
+#ifdef ENABLE_DBENGINE
+                    if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+                        rd->state->metric_uuid = mallocz(sizeof(uuid_t));
+                        uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid);
+                    }
+#endif
+                    rd->next = temp_rd;
+                    temp_rd = rd;
+                }
+                rrdset_unlock(st1);
+            }
+        }
+        rrdhost_unlock(st->rrdhost);
+    }
+
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, temp_rd);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -135,7 +195,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
             rrdr2ssv(r, wb, options, "", " ", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -148,7 +208,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
             rrdr2ssv(r, wb, options, "", ",", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -161,7 +221,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
             rrdr2ssv(r, wb, options, "[", ",", "]");
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
@@ -174,42 +234,42 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
-            rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
+            rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", ",", "\r\n", "");
+            rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd);
         }
         break;
 
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
-            rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
+            rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", "|", "\r\n", "");
+            rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd);
         }
         break;
 
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
             buffer_strcat(wb, "[\n");
-            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
+            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
         else {
             wb->contenttype = CT_APPLICATION_JSON;
             buffer_strcat(wb, "[\n");
-            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
+            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
         }
         break;
@@ -217,29 +277,29 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
-            rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
+            rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "");
+            rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd);
         }
         break;
 
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
-            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
+            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_HTML;
             buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
-            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
+            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "", temp_rd);
             buffer_strcat(wb, "</table>\n</center>\n</html>\n");
         }
         break;
@@ -248,9 +308,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
 
-        rrdr2json(r, wb, options, 1);
+        rrdr2json(r, wb, options, 1, temp_rd);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -260,9 +320,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
 
-        rrdr2json(r, wb, options, 1);
+        rrdr2json(r, wb, options, 1, temp_rd);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -271,9 +331,9 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
 
-        rrdr2json(r, wb, options, 0);
+        rrdr2json(r, wb, options, 0, temp_rd);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -284,15 +344,17 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd);
 
-        rrdr2json(r, wb, options, 0);
+        rrdr2json(r, wb, options, 0, temp_rd);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         break;
     }
 
+    free_temp_rrddim(temp_rd);
+
     rrdr_free(r);
     return HTTP_RESP_OK;
 }
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index 8fce4ef7b4..f235113480 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -65,6 +65,7 @@ extern int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
+        , char *context
 );
 
 extern int rrdset2value_api_v1(
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 68b3525715..6b46cc6bd2 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -281,7 +281,7 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI
 
 // ----------------------------------------------------------------------------
 
-static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims) {
+static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, RRDDIM *temp_rd) {
     rrdset_check_rdlock(r->st);
 
     if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
@@ -300,7 +300,7 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options,
 
     RRDDIM *d;
     long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
-    for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
+    for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) {
         if(    (match_ids   && simple_pattern_matches(pattern, d->id))
                || (match_names && simple_pattern_matches(pattern, d->name))
                 ) {
@@ -332,7 +332,7 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options,
         // but they are all zero
         // enable the selected ones
         // to avoid returning an empty chart
-        for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
+        for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next)
             if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED))
                 r->od[c] |= RRDR_DIMENSION_NONZERO;
     }
@@ -815,7 +815,8 @@ static RRDR *rrd2rrdr_fixedstep(
         , int update_every
         , time_t first_entry_t
         , time_t last_entry_t
-        , int absolute_period_requested
+        , int absolute_period_requested,
+        RRDDIM *temp_rd
 ) {
     int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
 
@@ -824,7 +825,7 @@ static RRDR *rrd2rrdr_fixedstep(
     long available_points = duration / update_every;
 
     if(duration <= 0 || available_points <= 0)
-        return rrdr_create(st, 1);
+        return rrdr_create(st, 1, temp_rd);
 
     // check the number of wanted points in the result
     if(unlikely(points_requested < 0)) points_requested = -points_requested;
@@ -982,7 +983,7 @@ static RRDR *rrd2rrdr_fixedstep(
     // initialize our result set
     // this also locks the chart for us
 
-    RRDR *r = rrdr_create(st, points_wanted);
+    RRDR *r = rrdr_create(st, points_wanted, temp_rd);
     if(unlikely(!r)) {
         #ifdef NETDATA_INTERNAL_CHECKS
         error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%l