/api/v2/X part 7 (#14797)

* /api/v2/weights, points key renamed to result

* /api/v2/weights, add node ids in response

* /api/v2/data remove NONZERO flag when all dimensions are zero and fix MIN/MAX grouping and statistics

* /api/v2/data expose view.dimensions.sts{}

* /api/v2 endpoints expose agents and additional info per node, that is needed to unify cloud responses

* /api/v2 nodes output now includes the duration of time spent per node

* jsonwrap view object renames and cleanup

* rework of the statistics returned by the query engine

* swagger work

* swagger work

* more swagger work

* updated swagger json

* added the remaining of the /api/v2 endpoints to swagger

* point.ar has been renamed point.arp

* updated weights endpoint

* fix compilation warnings

14 files changed, 3984 insertions, 3934 deletions

diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 700f804d34..8d7f49da9b 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -247,7 +247,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
 
 void rrdr2json_v2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     bool expose_gbc = query_target_aggregatable(qt);
 
@@ -268,7 +268,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
 
     buffer_json_member_add_object(wb, "point");
     buffer_json_member_add_uint64(wb, "value", 0);
-    buffer_json_member_add_uint64(wb, "ar", 1);
+    buffer_json_member_add_uint64(wb, "arp", 1);
     buffer_json_member_add_uint64(wb, "pa", 2);
     if(expose_gbc)
         buffer_json_member_add_uint64(wb, "count", 3);
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index d4c1e8a5ac..b8f47ea57d 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -101,7 +101,7 @@ struct summary_total_counts {
     size_t failed;
 };
 
-static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, struct query_metrics_counts *metrics) {
+static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) {
     if(unlikely(!totals || !metrics))
         return;
 
@@ -139,7 +139,7 @@ static inline void query_target_total_counts(BUFFER *wb, const char *key, struct
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_counts *metrics) {
+static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) {
     if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded)
         return;
 
@@ -160,7 +160,7 @@ static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_c
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_instance_counts(BUFFER *wb, struct query_instances_counts *instances) {
+static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) {
     if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded)
         return;
 
@@ -181,7 +181,7 @@ static inline void query_target_instance_counts(BUFFER *wb, struct query_instanc
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_counts *alerts, const char *name, bool array) {
+static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) {
     if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning)
         return;
 
@@ -208,40 +208,36 @@ static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_co
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, struct query_data_statistics *d) {
-    if(!d->group_points)
+static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) {
+    if(!sp->count)
         return;
 
     buffer_json_member_add_object(wb, "sts");
 
-    buffer_json_member_add_double(wb, "min", d->min);
-    buffer_json_member_add_double(wb, "max", d->max);
+    buffer_json_member_add_double(wb, "min", sp->min);
+    buffer_json_member_add_double(wb, "max", sp->max);
 
     if(query_target_aggregatable(qt)) {
-        buffer_json_member_add_uint64(wb, "cnt", d->group_points);
+        buffer_json_member_add_uint64(wb, "cnt", sp->count);
 
-        if(d->sum != 0.0)
-            buffer_json_member_add_double(wb, "sum", d->sum);
-
-        if(d->volume != 0.0)
-            buffer_json_member_add_double(wb, "vol", d->volume);
+        if(sp->sum != 0.0) {
+            buffer_json_member_add_double(wb, "sum", sp->sum);
+            buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt));
+        }
 
-        if(d->anomaly_sum != 0.0)
-            buffer_json_member_add_double(wb, "ars", d->anomaly_sum);
+        if(sp->anomaly_count != 0)
+            buffer_json_member_add_double(wb, "ars", storage_point_anomaly_rate(*sp));
     }
     else {
-//        buffer_json_member_add_double(wb, "min", d->min);
-//        buffer_json_member_add_double(wb, "max", d->max);
-
-        NETDATA_DOUBLE avg = (d->group_points) ? d->sum / (NETDATA_DOUBLE)d->group_points : 0.0;
+        NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0;
         if(avg != 0.0)
             buffer_json_member_add_double(wb, "avg", avg);
 
-        NETDATA_DOUBLE arp = (d->group_points) ? d->anomaly_sum / (NETDATA_DOUBLE)d->group_points : 0.0;
+        NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp);
         if(arp != 0.0)
             buffer_json_member_add_double(wb, "arp", arp);
 
-        NETDATA_DOUBLE con = (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0;
+        NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0;
         if(con != 0.0)
             buffer_json_member_add_double(wb, "con", con);
     }
@@ -254,15 +250,11 @@ static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const ch
         QUERY_NODE *qn = query_node(qt, c);
         RRDHOST *host = qn->rrdhost;
         buffer_json_add_array_item_object(wb);
-        buffer_json_member_add_uint64(wb, "ni", qn->slot);
-        buffer_json_member_add_string(wb, "mg", host->machine_guid);
-        if(qn->node_id[0])
-            buffer_json_member_add_string(wb, "nd", qn->node_id);
-        buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+        buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut);
         query_target_instance_counts(wb, &qn->instances);
         query_target_metric_counts(wb, &qn->metrics);
         query_target_alerts_counts(wb, &qn->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &qn->query_stats);
+        query_target_points_statistics(wb, qt, &qn->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &qn->metrics);
@@ -275,16 +267,16 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
     DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
 
     struct {
-        struct query_data_statistics query_stats;
-        struct query_instances_counts instances;
-        struct query_metrics_counts metrics;
-        struct query_alerts_counts alerts;
-    } x = { 0 }, *z;
+        STORAGE_POINT query_points;
+        QUERY_INSTANCES_COUNTS instances;
+        QUERY_METRICS_COUNTS metrics;
+        QUERY_ALERTS_COUNTS alerts;
+    } *z;
 
     for (long c = 0; c < (long) qt->contexts.used; c++) {
         QUERY_CONTEXT *qc = query_context(qt, c);
 
-        z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), &x, sizeof(x));
+        z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z));
 
         z->instances.selected += qc->instances.selected;
         z->instances.excluded += qc->instances.selected;
@@ -300,7 +292,7 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
         z->alerts.warning += qc->alerts.warning;
         z->alerts.critical += qc->alerts.critical;
 
-        query_target_merge_data_statistics(&z->query_stats, &qc->query_stats);
+        storage_point_merge_to(z->query_points, qc->query_points);
     }
 
     size_t unique_contexts = dictionary_entries(dict);
@@ -310,7 +302,7 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
         query_target_instance_counts(wb, &z->instances);
         query_target_metric_counts(wb, &z->metrics);
         query_target_alerts_counts(wb, &z->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &z->query_stats);
+                query_target_points_statistics(wb, qt, &z->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &z->metrics);
@@ -334,8 +326,7 @@ static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, cons
                   rrdinstance_acquired_id(qi->ria),
                   rrdinstance_acquired_name(qi->ria));
 
-        bool existing = 0;
-        bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
+        bool *set = dictionary_set(dict, name, NULL, sizeof(*set));
         if (!*set) {
             *set = true;
             buffer_json_add_array_item_array(wb);
@@ -369,7 +360,7 @@ static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, cons
 //            buffer_json_member_add_string(wb, "nd", qh->node_id);
         query_target_metric_counts(wb, &qi->metrics);
         query_target_alerts_counts(wb, &qi->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &qi->query_stats);
+        query_target_points_statistics(wb, qt, &qi->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &qi->metrics);
@@ -385,8 +376,8 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
     struct {
         const char *id;
         const char *name;
-        struct query_data_statistics query_stats;
-        struct query_metrics_counts metrics;
+        STORAGE_POINT query_points;
+        QUERY_METRICS_COUNTS metrics;
     } *z;
     size_t q = 0;
     for (long c = 0; c < (long) qt->dimensions.used; c++) {
@@ -426,7 +417,7 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
 
             if(qm->status & RRDR_DIMENSION_QUERIED) {
                 z->metrics.queried++;
-                query_target_merge_data_statistics(&z->query_stats, &qm->query_stats);
+                storage_point_merge_to(z->query_points, qm->query_points);
             }
         }
         else
@@ -440,7 +431,7 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
                         buffer_json_member_add_string(wb, "nm", z->name);
 
                     query_target_metric_counts(wb, &z->metrics);
-                    query_target_data_statistics(wb, qt, &z->query_stats);
+                    query_target_points_statistics(wb, qt, &z->query_points);
                     buffer_json_object_close(wb);
 
                     aggregate_into_summary_totals(totals, &z->metrics);
@@ -466,38 +457,34 @@ struct rrdlabels_formatting_v2 {
 struct rrdlabels_keys_dict_entry {
     const char *name;
     DICTIONARY *values;
-    struct query_data_statistics query_stats;
-    struct query_metrics_counts metrics;
+    STORAGE_POINT query_points;
+    QUERY_METRICS_COUNTS metrics;
 };
 
 struct rrdlabels_key_value_dict_entry {
     const char *key;
     const char *value;
-    struct query_data_statistics query_stats;
-    struct query_metrics_counts metrics;
+    STORAGE_POINT query_points;
+    QUERY_METRICS_COUNTS metrics;
 };
 
 static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
     struct rrdlabels_formatting_v2 *t = data;
 
-    struct rrdlabels_keys_dict_entry k = {
-            .name = name,
-            .values = NULL,
-            .metrics = (struct query_metrics_counts){ 0 },
-    }, *d = dictionary_set(t->keys, name, &k, sizeof(k));
-
-    if(!d->values)
+    struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d));
+    if(!d->values) {
+        d->name = name;
         d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+    }
 
     char n[RRD_ID_LENGTH_MAX * 2 + 2];
     snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
 
-    struct rrdlabels_key_value_dict_entry x = {
-            .key = name,
-            .value = value,
-            .query_stats = (struct query_data_statistics) { 0 },
-            .metrics = (struct query_metrics_counts){ 0 },
-    }, *z = dictionary_set(d->values, n, &x, sizeof(x));
+    struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z));
+    if(!z->key) {
+        z->key = name;
+        z->value = value;
+    }
 
     if(t->v2) {
         QUERY_INSTANCE *qi = t->qi;
@@ -512,8 +499,8 @@ static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL
         d->metrics.queried += qi->metrics.queried;
         d->metrics.failed += qi->metrics.failed;
 
-        query_target_merge_data_statistics(&z->query_stats, &qi->query_stats);
-        query_target_merge_data_statistics(&d->query_stats, &qi->query_stats);
+        storage_point_merge_to(z->query_points, qi->query_points);
+        storage_point_merge_to(d->query_points, qi->query_points);
     }
 
     return 1;
@@ -537,7 +524,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
                     buffer_json_add_array_item_object(wb);
                     buffer_json_member_add_string(wb, "id", d_dfe.name);
                     query_target_metric_counts(wb, &d->metrics);
-                    query_target_data_statistics(wb, qt, &d->query_stats);
+                    query_target_points_statistics(wb, qt, &d->query_points);
                     aggregate_into_summary_totals(key_totals, &d->metrics);
                     buffer_json_member_add_array(wb, "vl");
                 }
@@ -547,7 +534,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
                                 buffer_json_add_array_item_object(wb);
                                 buffer_json_member_add_string(wb, "id", z->value);
                                 query_target_metric_counts(wb, &z->metrics);
-                                query_target_data_statistics(wb, qt, &z->query_stats);
+                                query_target_points_statistics(wb, qt, &z->query_points);
                                 buffer_json_object_close(wb);
                                 aggregate_into_summary_totals(value_totals, &z->metrics);
                             } else {
@@ -571,7 +558,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
 
 static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
     buffer_json_member_add_array(wb, key);
-    struct query_alerts_counts x = { 0 }, *z;
+    QUERY_ALERTS_COUNTS *z;
 
     DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
     for (long c = 0; c < (long) qt->instances.used; c++) {
@@ -581,7 +568,7 @@ static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const c
             netdata_rwlock_rdlock(&st->alerts.rwlock);
             if (st->alerts.base) {
                 for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
-                    z = dictionary_set(dict, string2str(rc->name), &x, sizeof(x));
+                    z = dictionary_set(dict, string2str(rc->name), NULL, sizeof(*z));
 
                     switch(rc->status) {
                         case RRDCALC_STATUS_CLEAR:
@@ -720,52 +707,102 @@ static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *k
     return i;
 }
 
-static inline void rrdr_dimension_view_average_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dv)
+static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) {
+    STORAGE_POINT *sp = (dview) ? r->dview : r->dqp;
+    NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0;
+
+    if(unlikely(!sp))
         return;
 
-    buffer_json_member_add_array(wb, key);
+    if(key)
+        buffer_json_member_add_object(wb, key);
 
+    buffer_json_member_add_array(wb, "min");
     for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+        if (!rrdr_dimension_should_be_exposed(r->od[c], options))
             continue;
 
-        buffer_json_add_array_item_double(wb, r->dv[c]);
+        buffer_json_add_array_item_double(wb, sp[c].min);
     }
+    buffer_json_array_close(wb);
+
+    buffer_json_member_add_array(wb, "max");
+    for(size_t c = 0; c < r->d ; c++) {
+        if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
 
+        buffer_json_add_array_item_double(wb, sp[c].max);
+    }
     buffer_json_array_close(wb);
-}
 
-static inline void rrdr_dimension_view_minimum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dmin)
-        return;
+    if(options & RRDR_OPTION_RETURN_RAW) {
+        buffer_json_member_add_array(wb, "sum");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_member_add_array(wb, key);
+            buffer_json_add_array_item_double(wb, sp[c].sum);
+        }
+        buffer_json_array_close(wb);
 
-    for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
-            continue;
+        buffer_json_member_add_array(wb, "cnt");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            buffer_json_add_array_item_uint64(wb, sp[c].count);
+        }
+        buffer_json_array_close(wb);
 
-        buffer_json_add_array_item_double(wb, r->dmin[c]);
+        buffer_json_member_add_array(wb, "ars");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            buffer_json_add_array_item_uint64(wb, sp[c].anomaly_count * 100 / anomaly_rate_multiplier);
+        }
+        buffer_json_array_close(wb);
     }
+    else {
+        NETDATA_DOUBLE sum = 0.0;
+        for(size_t c = 0; c < r->d ; c++) {
+            if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_array_close(wb);
-}
+            sum += ABS(sp[c].sum);
+        }
 
-static inline void rrdr_dimension_view_maximum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dmin)
-        return;
+        buffer_json_member_add_array(wb, "avg");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_member_add_array(wb, key);
+            buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c]));
+        }
+        buffer_json_array_close(wb);
 
-    for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
-            continue;
+        buffer_json_member_add_array(wb, "arp");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-        buffer_json_add_array_item_double(wb, r->dmax[c]);
+            buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier);
+        }
+        buffer_json_array_close(wb);
+
+        buffer_json_member_add_array(wb, "con");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0;
+            buffer_json_add_array_item_double(wb, con);
+        }
+        buffer_json_array_close(wb);
     }
 
-    buffer_json_array_close(wb);
+    if(key)
+        buffer_json_object_close(wb);
 }
 
 static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
@@ -775,8 +812,7 @@ static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
     buffer_json_member_add_object(wb, key);
     buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(qt->timings.preprocessed_ut - qt->timings.received_ut) / USEC_PER_MS);
     buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(qt->timings.executed_ut - qt->timings.preprocessed_ut) / USEC_PER_MS);
-    buffer_json_member_add_double(wb, "group_by_ms", (NETDATA_DOUBLE)(qt->timings.group_by_ut - qt->timings.executed_ut) / USEC_PER_MS);
-    buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.group_by_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.executed_ut) / USEC_PER_MS);
     buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.received_ut) / USEC_PER_MS);
     buffer_json_object_close(wb);
 }
@@ -784,7 +820,7 @@ static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     long rows = rrdr_rows(r);
 
@@ -812,7 +848,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     buffer_json_member_add_time_t(wb, "after", r->view.after);
     buffer_json_member_add_time_t(wb, "before", r->view.before);
     buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method));
-    web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
+    web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options);
 
     if(!rrdr_dimension_names(wb, "dimension_names", r, options))
         rows = 0;
@@ -820,7 +856,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     if(!rrdr_dimension_ids(wb, "dimension_ids", r, options))
         rows = 0;
 
-    if (r->view.options & RRDR_OPTION_ALL_DIMENSIONS) {
+    if (options & RRDR_OPTION_ALL_DIMENSIONS) {
         query_target_summary_instances_v1(wb, qt, "full_chart_list");
         query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL);
         query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL);
@@ -1006,8 +1042,7 @@ static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
 
         size_t added = 0;
         for(size_t c = 0; c < qt->contexts.used ;c++) {
-            bool old = false;
-            bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), &old, sizeof(old));
+            bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set));
             if(!*set) {
                 *set = true;
                 if(added)
@@ -1148,7 +1183,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
                                 buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name));
                             }
 
-                            query_target_data_statistics(wb, qt, &qm->query_stats);
+                            query_target_points_statistics(wb, qt, &qm->query_points);
 
                             if(options & RRDR_OPTION_DEBUG)
                                 jsonwrap_query_metric_plan(wb, qm);
@@ -1190,7 +1225,7 @@ void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {
 
 void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     char kq[2] = "\"",                    // key quote
          sq[2] = "\"";                    // string quote
@@ -1201,8 +1236,8 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
     }
 
     buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
-
     buffer_json_member_add_uint64(wb, "api", 2);
+    buffer_json_agents_array_v2(wb, 0);
 
     if(options & RRDR_OPTION_DEBUG) {
         buffer_json_member_add_string(wb, "id", qt->id);
@@ -1416,25 +1451,32 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) {
 void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     buffer_json_member_add_object(wb, "view");
     {
         query_target_title(wb, qt, r->internal.contexts);
-        buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
-        web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
-        buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
         buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
         buffer_json_member_add_time_t(wb, "after", r->view.after);
         buffer_json_member_add_time_t(wb, "before", r->view.before);
 
-        buffer_json_member_add_object(wb, "partial_data_trimming");
-        buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
-        buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
-        buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
-        buffer_json_object_close(wb);
+        if(options & RRDR_OPTION_DEBUG) {
+            buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
+            web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options);
+            buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
+        }
+
+        if(options & RRDR_OPTION_DEBUG) {
+            buffer_json_member_add_object(wb, "partial_data_trimming");
+            buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
+            buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
+            buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
+            buffer_json_object_close(wb);
+        }
+
+        if(options & RRDR_OPTION_RETURN_RAW)
+            buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
 
-        buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
         query_target_combined_units_v2(wb, qt, r->internal.contexts);
         query_target_combined_chart_type(wb, qt, r->internal.contexts);
         buffer_json_member_add_object(wb, "dimensions");
@@ -1445,11 +1487,8 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
             rrdr_dimension_units_array_v2(wb, "units", r, options);
             rrdr_dimension_priority_array_v2(wb, "priorities", r, options);
             rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options);
-            rrdr_dimension_view_minimum_values(wb, "view_minimum_values", r, options);
-            rrdr_dimension_view_maximum_values(wb, "view_maximum_values", r, options);
-            rrdr_dimension_view_average_values(wb, "view_average_values", r, options);
-            size_t dims = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options);
-            buffer_json_member_add_uint64(wb, "count", dims);
+            rrdr_dimension_query_points_statistics(wb, NULL, r, options, true);
+            rrdr_dimension_query_points_statistics(wb, "sts", r, options, false);
             rrdr_json_group_by_labels(wb, "labels", r, options);
         }
         buffer_json_object_close(wb); // dimensions
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index af1cfe93e2..7d727ce1a1 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -4,8 +4,8 @@
 #include "database/storage_engine.h"
 
 inline bool query_target_has_percentage_units(struct query_target *qt) {
-    if(qt->request.options & RRDR_OPTION_PERCENTAGE ||
-       qt->request.time_group_method == RRDR_GROUPING_CV)
+    if(qt->window.options & RRDR_OPTION_PERCENTAGE ||
+       qt->window.time_group_method == RRDR_GROUPING_CV)
         return true;
 
     return false;
@@ -170,7 +170,6 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
     }
 
     RRDR *r = rrd2rrdr(owa, qt);
-    qt->timings.executed_ut = now_monotonic_usec();
 
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
@@ -191,9 +190,7 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
         *latest_timestamp = r->view.before;
 
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
-
-    qt->timings.group_by_ut = now_monotonic_usec();
+    RRDR_OPTIONS options = qt->window.options;
 
     switch(format) {
     case DATASOURCE_SSV:
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index 85c5446ee3..7dbcf8bf05 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -63,7 +63,7 @@ void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTION
 
 struct query_target;
 bool query_target_has_percentage_units(struct query_target *qt);
-#define query_target_aggregatable(qt) (qt->request.options & RRDR_OPTION_RETURN_RAW)
+#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW)
 
 int rrdset2value_api_v1(
         RRDSET *st
diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c
index e6a663a431..ce48c343ca 100644
--- a/web/api/formatters/value/value.c
+++ b/web/api/formatters/value/value.c
@@ -107,7 +107,8 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
                         .max = NAN,
                         .sum = NAN,
                         .anomaly_count = 0,
-                }
+                },
+                .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0,
         };
     }
     else {
@@ -118,11 +119,16 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
                 .result_points = r->stats.result_points_generated,
                 .sp = {
                         .count = 0,
-                }
+                },
+                .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut,
         };
 
-        for(size_t d = 0; d < r->d ;d++)
-            storage_point_merge_to(qv.sp, r->drs[d]);
+        for(size_t d = 0; d < r->internal.qt->query.used ;d++) {
+            if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options))
+                continue;
+
+            storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points);
+        }
 
         for(size_t t = 0; t < storage_tiers ;t++)
             qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;
diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h
index 7ff613cc03..072ca14f8a 100644
--- a/web/api/formatters/value/value.h
+++ b/web/api/formatters/value/value.h
@@ -14,6 +14,7 @@ typedef struct storage_value {
     size_t storage_points_per_tier[RRD_STORAGE_TIERS];
     size_t result_points;
     STORAGE_POINT sp;
+    usec_t duration_ut;
 } QUERY_VALUE;
 
 struct rrdmetric_acquired;
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 275001be9b..5db5149d4d 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -3,12 +3,210 @@
   "info": {
     "title": "Netdata API",
     "description": "Real-time performance and health monitoring.",
-    "version": "1.38"
+    "version": "1.38",
+    "contact": {
+      "name": "Netdata Agent API",
+      "email": "info@netdata.cloud",
+      "url": "https://netdata