/api/v1/weights endpoint (#13449)

* /api/v1/weights endpoints

* high resolution anomaly rate in parallel with queries; points and options in /api/v1/weights reflect the truth

* context printing

* merged metric_correlations with weights API; added parameter tier to select the tier to run the query; weight api now returns points per tier; added swagger info about weights api

* moved metric_correlations files to web/api/queries as weights

* added contexts filtering; renamed correlated_dimensions; weights API is always enabled; code cleanup

* allow returning zero results

24 files changed, 730 insertions, 181 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0babcc11b1..5ef3a297d1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -774,8 +774,6 @@ set(RRD_PLUGIN_FILES
         database/engine/metadata_log/compaction.h
         database/KolmogorovSmirnovDist.c
         database/KolmogorovSmirnovDist.h
-        database/metric_correlations.c
-        database/metric_correlations.h
         )
 
 set(WEB_PLUGIN_FILES
@@ -822,6 +820,8 @@ set(API_PLUGIN_FILES
         web/api/queries/ses/ses.h
         web/api/queries/des/des.c
         web/api/queries/des/des.h
+        web/api/queries/weights.c
+        web/api/queries/weights.h
         web/api/formatters/rrd2json.c
         web/api/formatters/rrd2json.h
         web/api/formatters/csv/csv.c
diff --git a/Makefile.am b/Makefile.am
index 5eefdc13cf..b962bb7973 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -473,8 +473,6 @@ RRD_PLUGIN_FILES = \
     database/sqlite/sqlite3.h \
     database/KolmogorovSmirnovDist.c \
     database/KolmogorovSmirnovDist.h \
-    database/metric_correlations.c \
-    database/metric_correlations.h \
     $(NULL)
 
 database/sqlite/sqlite3.$(OBJEXT) : CFLAGS += -Wno-cast-function-type
@@ -599,6 +597,8 @@ API_PLUGIN_FILES = \
     web/api/queries/stddev/stddev.h \
     web/api/queries/sum/sum.c \
     web/api/queries/sum/sum.h \
+    web/api/queries/weights.c \
+    web/api/queries/weights.h \
     web/api/formatters/rrd2json.c \
     web/api/formatters/rrd2json.h \
     web/api/formatters/csv/csv.c \
diff --git a/configure.ac b/configure.ac
index 377d08d6ab..9dbb6a7c1a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1681,6 +1681,8 @@ AC_CONFIG_FILES([
     database/Makefile
     database/engine/Makefile
     database/engine/metadata_log/Makefile
+    database/ram/Makefile
+    database/sqlite/Makefile
     diagrams/Makefile
     exporting/Makefile
     exporting/graphite/Makefile
diff --git a/daemon/common.h b/daemon/common.h
index da96e2ac1c..2a45ffe70e 100644
--- a/daemon/common.h
+++ b/daemon/common.h
@@ -84,9 +84,6 @@
 #include "commands.h"
 #include "analytics.h"
 
-// metric correlations
-#include "database/metric_correlations.h"
-
 // global netdata daemon variables
 extern char *netdata_configured_hostname;
 extern char *netdata_configured_user_config_dir;
diff --git a/daemon/main.c b/daemon/main.c
index f60cd34e2b..a2317c1b72 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -713,7 +713,9 @@ static void get_netdata_configured_variables() {
     // metric correlations
 
     enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
-    default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method)));
+    default_metric_correlations_method = weights_string_to_method(config_get(
+        CONFIG_SECTION_GLOBAL, "metric correlations method",
+        weights_method_to_string(default_metric_correlations_method)));
 
     // --------------------------------------------------------------------
 
diff --git a/database/Makefile.am b/database/Makefile.am
index 9ab85b4e74..dc87e61bb8 100644
--- a/database/Makefile.am
+++ b/database/Makefile.am
@@ -5,6 +5,8 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
 SUBDIRS = \
     engine \
+    ram \
+    sqlite \
     $(NULL)
 
 dist_noinst_DATA = \
diff --git a/database/metric_correlations.h b/database/metric_correlations.h
deleted file mode 100644
index 4d0b4dba3b..0000000000
--- a/database/metric_correlations.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_METRIC_CORRELATIONS_H
-#define NETDATA_METRIC_CORRELATIONS_H 1
-
-#include "web/api/queries/query.h"
-
-typedef enum {
-    METRIC_CORRELATIONS_KS2    = 1,
-    METRIC_CORRELATIONS_VOLUME = 2,
-} METRIC_CORRELATIONS_METHOD;
-
-extern int enable_metric_correlations;
-extern int metric_correlations_version;
-extern METRIC_CORRELATIONS_METHOD default_metric_correlations_method;
-
-extern int metric_correlations (RRDHOST *host, BUFFER *wb, METRIC_CORRELATIONS_METHOD method,
-                               RRDR_GROUPING group, const char *group_options,
-                               long long baseline_after, long long baseline_before,
-                               long long after, long long before,
-                               long long points, RRDR_OPTIONS options, int timeout);
-
-extern METRIC_CORRELATIONS_METHOD mc_string_to_method(const char *method);
-extern const char *mc_method_to_string(METRIC_CORRELATIONS_METHOD method);
-extern int mc_unittest(void);
-
-#endif //NETDATA_METRIC_CORRELATIONS_H
diff --git a/database/ram/Makefile.am b/database/ram/Makefile.am
new file mode 100644
index 0000000000..59250a997e
--- /dev/null
+++ b/database/ram/Makefile.am
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+    $(NULL)
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)
diff --git a/database/ram/README.md b/database/ram/README.md
new file mode 100644
index 0000000000..73562f0fff
--- /dev/null
+++ b/database/ram/README.md
@@ -0,0 +1,7 @@
+<!--
+title: "RAM database modes"
+description: "Netdata's RAM database modes."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/database/ram/README.md
+-->
+
+# RAM modes
\ No newline at end of file
diff --git a/health/health.c b/health/health.c
index f4b0665eb7..01ab6b918a 100644
--- a/health/health.c
+++ b/health/health.c
@@ -894,7 +894,7 @@ void *health_main(void *ptr) {
                                                   rc->after, rc->before, rc->group, NULL,
                                                   0, rc->options,
                                                   &rc->db_after,&rc->db_before,
-                                                  NULL, NULL,
+                                                  NULL, NULL, NULL,
                                                   &value_is_null, NULL, 0, 0);
 
                     if (unlikely(ret != 200)) {
diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index 14759863d5..00b4ad650a 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -1110,7 +1110,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
                                       (dimensions) ? buffer_tostring(dimensions) : NULL,
                                       points, after, before, group, group_options, 0, options,
                                       NULL, &latest_timestamp,
-                                      NULL, NULL,
+                                      NULL, NULL, NULL,
                                       &value_is_null, NULL, 0, 0);
 
         // if the value cannot be calculated, show empty badge
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index bf9bf9a3d0..6f07b9aa43 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -162,7 +162,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
     for(i = start; i != end ;i += step) {
         NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
         RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
-        uint8_t *ar = &r->ar[ i * r->d ];
+        NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
 
         time_t now = r->t[i];
 
@@ -225,7 +225,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
             for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
                 NETDATA_DOUBLE n;
                 if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
-                    n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
+                    n = ar[c];
                 else
                     n = cn[c];
 
@@ -246,7 +246,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
 
             NETDATA_DOUBLE n;
             if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
-                n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
+                n = ar[c];
             else
                 n = cn[c];
 
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 77e002b297..7aa478d95a 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -193,9 +193,10 @@ int rrdset2value_api_v1(
         , time_t *db_after
         , time_t *db_before
         , size_t *db_points_read
+        , size_t *db_points_per_tier
         , size_t *result_points_generated
         , int *value_is_null
-        , uint8_t *anomaly_rate
+        , NETDATA_DOUBLE *anomaly_rate
         , int timeout
         , int tier
 ) {
@@ -216,6 +217,11 @@ int rrdset2value_api_v1(
     if(db_points_read)
         *db_points_read += r->internal.db_points_read;
 
+    if(db_points_per_tier) {
+        for(int t = 0; t < storage_tiers ;t++)
+            db_points_per_tier[t] += r->internal.tier_points_read[t];
+    }
+
     if(result_points_generated)
         *result_points_generated += r->internal.result_points_generated;
 
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index fa82463e17..6be53ff8a6 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -96,9 +96,10 @@ extern int rrdset2value_api_v1(
         , time_t *db_after
         , time_t *db_before
         , size_t *db_points_read
+        , size_t *db_points_per_tier
         , size_t *result_points_generated
         , int *value_is_null
-        , uint8_t *anomaly_rate
+        , NETDATA_DOUBLE *anomaly_rate
         , int timeout
         , int tier
 );
diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c
index 84d8e11ef6..30e00c0689 100644
--- a/web/api/formatters/value/value.c
+++ b/web/api/formatters/value/value.c
@@ -3,19 +3,19 @@
 #include "value.h"
 
 
-inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
+inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd) {
     long c;
     RRDDIM *d;
 
     NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
     RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
-    uint8_t *ar = &r->ar[ i * r->d ];
+    NETDATA_DOUBLE *ar = &r->ar[ i * r->d ];
 
     NETDATA_DOUBLE sum = 0, min = 0, max = 0, v;
     int all_null = 1, init = 1;
 
     NETDATA_DOUBLE total = 1;
-    size_t total_anomaly_rate = 0;
+    NETDATA_DOUBLE total_anomaly_rate = 0;
 
     int set_min_max = 0;
     if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h
index 7790e3956a..fc1c7bf08d 100644
--- a/web/api/formatters/value/value.h
+++ b/web/api/formatters/value/value.h
@@ -5,7 +5,6 @@
 
 #include "../rrd2json.h"
 
-extern NETDATA_DOUBLE
-rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd);
+extern NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd);
 
 #endif //NETDATA_API_FORMATTER_VALUE_H
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index 1c80663741..c6282c13c8 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -1100,7 +1100,8 @@ paths:
   /metric_correlations:
     get:
       summary: "Analyze all the metrics to find their correlations"
-      description: "Given two time-windows (baseline, highlight), it goes
+      description: "THIS ENDPOINT IS OBSOLETE. Use the /weights endpoint.
+        Given two time-windows (baseline, highlight), it goes
         through all the available metrics, querying both windows and tries to find
         how these two windows relate to each other. It supports
         multiple algorithms to do so. The result is a list of all
@@ -1262,6 +1263,181 @@ paths:
             that correlated the metrics did not produce any result.
         "504":
           description: Timeout - the query took too long and has been cancelled.
+  /weights:
+    get:
+      summary: "Analyze all the metrics using an algorithm and score them accordingly"
+      description: "This endpoint goes through all metrics and scores them according to an algorithm."
+      parameters:
+        - name: baseline_after
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            starting point of baseline window, or a relative number of
+            seconds (negative, relative to parameter baseline_before). Netdata will
+            assume it is a relative number if it is less that 3 years (in seconds).
+            This parameter is used in KS2 and VOLUME algorithms.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: -300
+        - name: baseline_before
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            ending point of the baseline window, or a relative number of
+            seconds (negative), relative to the last collected timestamp.
+            Netdata will assume it is a relative number if it is less than 3
+            years (in seconds).
+            This parameter is used in KS2 and VOLUME algorithms.
+          required: false
+          schema:
+            type: number
+            format: integer
+            default: -60
+        - name: after
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            starting point of highlighted window, or a relative number of
+            seconds (negative, relative to parameter highlight_before). Netdata will
+            assume it is a relative number if it is less that 3 years (in seconds).
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: -60
+        - name: before
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            ending point of the highlighted window, or a relative number of
+            seconds (negative), relative to the last collected timestamp.
+            Netdata will assume it is a relative number if it is less than 3
+            years (in seconds).
+          required: false
+          schema:
+            type: number
+            format: integer
+            default: 0
+        - name: context
+          in: query
+          description: A simple pattern matching the contexts to evaluate.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: string
+        - name: points
+          in: query
+          description: The number of points to be evaluated for the highlighted window.
+            The baseline window will be adjusted automatically to receive a proportional
+            amount of points.
+            This parameter is only used by the KS2 algorithm.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: 500
+        - name: method
+          in: query
+          description: the algorithm to run
+          required: false
+          schema:
+            type: string
+            enum:
+              - ks2
+              - volume
+              - anomaly-rate
+            default: anomaly-rate
+        - name: tier
+          in: query
+          description: Use the specified database tier
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+        - name: timeout
+          in: query
+          description: Cancel the query if to takes more that this amount of milliseconds.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: 60000
+        - name: options
+          in: query
+          description: Options that affect data generation.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: array
+            items:
+              type: string
+              enum:
+                - min2max
+                - abs
+                - absolute
+                - absolute-sum
+                - null2zero
+                - percentage
+                - unaligned
+                - nonzero
+                - anomaly-bit
+                - raw
+            default:
+              - null2zero
+              - nonzero
+              - unaligned
+        - name: group
+          in: query
+          description: The grouping method. If multiple collected values are to be grouped
+            in order to return fewer points, this parameters defines the method
+            of grouping. methods supported "min", "max", "average", "sum",
+            "incremental-sum". "max" is actually calculated on the absolute
+            value collected (so it works for both positive and negative
+            dimensions to return the most extreme value in either direction).
+          required: true
+          allowEmptyValue: false
+          schema:
+            type: string
+            enum:
+              - min
+              - max
+              - average
+              - median
+              - stddev
+              - sum
+              - incremental-sum
+              - ses
+              - des
+              - cv
+              - countif
+            default: average
+        - name: group_options
+          in: query
+          description: When the group function supports additional parameters, this field
+            can be used to pass them to it. Currently only "countif" supports this.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: string
+      responses:
+        "200":
+          description: JSON object with weights for each context, chart and dimension.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/weight"
+        "400":
+          description: The given parameters are invalid.
+        "403":
+          description: metrics correlations are not enabled on this Netdata Agent.
+        "404":
+          description: No charts could be found, or the method
+            that correlated the metrics did not produce any result.
+        "504":
+          description: Timeout - the query took too long and has been cancelled.
 servers:
   - url: https://registry.my-netdata.io/api/v1
   - url: http://registry.my-netdata.io/api/v1
@@ -2197,3 +2373,81 @@ components:
                       type: number
                     dimension2-name:
                       type: number
+    weight:
+      type: object
+      properties:
+        after:
+          description: the start time of the highlighted window
+          type: integer
+        before:
+          description: the end time of the highlighted window
+          type: integer
+        duration:
+          description: the duration of the highlighted window
+          type: integer
+        points:
+          description: the points of the highlighted window
+          type: integer
+        baseline_after:
+          description: the start time of the baseline window
+          type: integer
+        baseline_before:
+          description: the end time of the baseline window
+          type: integer
+        baseline_duration:
+          description: the duration of the baseline window
+          type: integer
+        baseline_points:
+          description: the points of the baseline window
+          type: integer
+        group:
+          description: the grouping method across time
+          type: string
+        method:
+          description: the correlation method used
+          type: string
+        options:
+          description: a comma separated list of the query options set
+          type: string
+        correlated_dimensions:
+          description: the number of dimensions returned in the result
+        total_dimensions_count:
+          description: the total number of dimensions evaluated
+          type: integer
+        statistics:
+          type: object
+          properties:
+            query_time_ms:
+              type: number
+            db_queries:
+              type: integer
+            db_points_read:
+              type: integer
+            query_result_points:
+              type: integer
+            binary_searches:
+              type: integer
+        contexts:
+          type: object
+          description: An object containing context objects.
+          properties:
+            contextX:
+              type: object
+              properties:
+                charts:
+                  type: object
+                  properties:
+                    chartX:
+                      type: object
+                      properties:
+                        dimensions:
+                          type: object
+                          properties:
+                            dimensionX:
+                              type: number
+                        weight:
+                          description: The average chart weight
+                          type: number
+                      weight:
+                        description: The average context weight
+                        type: number
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 40bc11426e..0247075b27 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -567,7 +567,7 @@ typedef struct query_point {
     time_t end_time;
     time_t start_time;
     NETDATA_DOUBLE value;
-    size_t anomaly;
+    NETDATA_DOUBLE anomaly;
     SN_FLAGS flags;
 #ifdef NETDATA_INTERNAL_CHECKS
     size_t id;
@@ -628,7 +628,7 @@ typedef struct query_engine_ops {
     NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
     size_t group_points_non_zero;
     size_t group_points_added;
-    size_t group_anomaly_rate;
+    NETDATA_DOUBLE group_anomaly_rate;
     RRDR_VALUE_FLAGS group_value_flags;
 
     // statistics
@@ -910,14 +910,14 @@ static inline void rrd2rrdr_do_dimension(
 
                 new_point.start_time = sp.start_time;
                 new_point.end_time   = sp.end_time;
-                new_point.anomaly    = sp.count ? sp.anomaly_count * 100 / sp.count : 0;
+                new_point.anomaly    = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
                 query_point_set_id(new_point, ops.db_total_points_read);
 
                 // set the right value to the point we got
                 if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
 
                     if(unlikely(use_anomaly_bit_as_value))
-                        new_point.value = (NETDATA_DOUBLE)new_point.anomaly;
+                        new_point.value =  new_point.anomaly;
 
                     else {
                         switch (ops.tier_query_fetch) {
@@ -1073,8 +1073,7 @@ static inline void rrd2rrdr_do_dimension(
             // we only store uint8_t anomaly rates,
             // so let's get double precision by storing
             // anomaly rates in the range 0 - 200
-            ops.group_anomaly_rate = (ops.group_anomaly_rate << 1) / ops.group_points_added;
-            r->ar[rrdr_o_v_index] = (uint8_t)ops.group_anomaly_rate;
+            r->ar[rrdr_o_v_index] = ops.group_anomaly_rate / (NETDATA_DOUBLE)ops.group_points_added;
 
             if(likely(points_added || dim_id_in_rrdr)) {
                 // find the min/max across all dimensions
diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c
index 954808a195..ecf4ca2ac3 100644
--- a/web/api/queries/rrdr.c
+++ b/web/api/queries/rrdr.c
@@ -82,7 +82,7 @@ RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points
     r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
     r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
     r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
-    r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(uint8_t));
+    r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
     r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
 
     r->group = 1;
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index 5c2598c439..1c80e103fa 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -83,7 +83,7 @@ typedef struct rrdresult {
     time_t *t;                // array of n timestamps
     NETDATA_DOUBLE *v;        // array n x d values
     RRDR_VALUE_FLAGS *o;      // array n x d options for each value returned
-    uint8_t *ar;              // array n x d of anomaly rates (0 - 200)
+    NETDATA_DOUBLE *ar;       // array n x d of anomaly rates (0 - 100)
 
     long group;               // how many collected values were grouped for each row
     int update_every;         // what is the suggested update frequency in seconds
diff --git a/database/metric_correlations.c b/web/api/queries/weights.c
index 01920da006..97a00f91cf 100644
--- a/database/metric_correlations.c
+++ b/web/api/queries/weights.c
@@ -1,45 +1,47 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "daemon/common.h"
-#include "KolmogorovSmirnovDist.h"
+#include "database/KolmogorovSmirnovDist.h"
 
 #define MAX_POINTS 10000
 int enable_metric_correlations = CONFIG_BOOLEAN_YES;
 int metric_correlations_version = 1;
-METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_KS2;
+WEIGHTS_METHOD default_metric_correlations_method = WEIGHTS_METHOD_MC_KS2;
 
-typedef struct mc_stats {
+typedef struct weights_stats {
     NETDATA_DOUBLE max_base_high_ratio;
     size_t db_points;
     size_t result_points;
     size_t db_queries;
+    size_t db_points_per_tier[RRD_STORAGE_TIERS];
     size_t binary_searches;
-} MC_STATS;
+} WEIGHTS_STATS;
 
 // ----------------------------------------------------------------------------
 // parse and render metric correlations methods
 
 static struct {
     const char *name;
-    METRIC_CORRELATIONS_METHOD value;
-} metric_correlations_methods[] = {
-      { "ks2"         , METRIC_CORRELATIONS_KS2 }
-    , { "volume"      , METRIC_CORRELATIONS_VOLUME }
-    , { NULL          , 0 }
+    WEIGHTS_METHOD value;
+} weights_methods[] = {
+      { "ks2"          , WEIGHTS_METHOD_MC_KS2}
+    , { "volume"       , WEIGHTS_METHOD_MC_VOLUME}
+    , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE}
+    , { NULL           , 0 }
 };
 
-METRIC_CORRELATIONS_METHOD mc_string_to_method(const char *method) {
-    for(int i = 0; metric_correlations_methods[i].name ;i++)
-        if(strcmp(method, metric_correlations_methods[i].name) == 0)
-            return metric_correlations_methods[i].value;
+WEIGHTS_METHOD weights_string_to_method(const char *method) {
+    for(int i = 0; weights_methods[i].name ;i++)
+        if(strcmp(method, weights_methods[i].name) == 0)
+            return weights_methods[i].value;
 
     return default_metric_correlations_method;
 }