73x times faster metrics correlations at the agent (#13107)

* faster correlations

* 4x times faster correlations

* a little bit more help

* 10x times faster metrics correlations

* 6 digits precision; better comments

* enabled metrics correlations by default

* abstracted DIFFS_NUMBER to allow easily changing it

* reworked the entire logic to have more accuracy and support a baseline that is power of two multiple of highlight

* properly calculate shifts

* even more improved version

* added support for timeout; fixed another memory leak; skipped hidden dimensions

* default timeout 1min

* reduce memory even further

* use dictionary for the list of charts and optimize locks

* return 403 forbidden, when mc is not enabled

* added query options

* dont process zero dimensions

* added volume method as an option to metric correlations ; now metric correlations can support multiple implementations

* make sure we will never crash

* spread results evenly for both kstwo and volume

* fixed bug in query engine that was missing misaligned queries when a single point was requested from the db; improved comments; improved query flags

* updated swagger and added sane defaults; query options are now supported, including anomaly-bit

* added "raw" option to allow cross node correlations; added "group" option to allow different time aggregations; allowed calling metric correlations without any parameters; allowed calling metric correlations with relative timestamps; added timeout to volume method; properly handled timeout on ks2 method; json output now sends all parameters back - same for json_wrap; modified query engine to use present time for relative timestamps; modified "allow_past" to mean both past backwards and forwards

* emulate the old behaviour about zero points

* 100% accuracy against python ks_2samp(); now the default is volume and the default points are 500

* added config option to change default metric correlations method

* removed work-arounds now that rrdlabels are merged

14 files changed, 581 insertions, 128 deletions

diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index 65ca21d196..ed317508be 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -1101,8 +1101,12 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
 
         // if the collected value is too old, don't calculate its value
         if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
-            ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
-                                      , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null, 0);
+            ret = rrdset2value_api_v1(st, w->response.data, &n,
+                                      (dimensions) ? buffer_tostring(dimensions) : NULL,
+                                      points, after, before, group, 0, options,
+                                      NULL, &latest_timestamp,
+                                      NULL, NULL,
+                                      &value_is_null, 0);
 
         // if the value cannot be calculated, show empty badge
         if (ret != HTTP_RESP_OK) {
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index af097a6712..ed8cbf00ce 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -35,7 +35,7 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL
 }
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    QUERY_PARAMS *rrdset_query_data)
+    RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data)
 {
     struct context_param *context_param_list = rrdset_query_data->context_param_list;
     char *chart_label_key = rrdset_query_data->chart_label_key;
@@ -76,7 +76,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %slast_entry%s: %u,\n"
                        "   %sbefore%s: %u,\n"
                        "   %safter%s: %u,\n"
-                       "   %sdimension_names%s: ["
+                       "   %sgroup%s: %s%s%s,\n"
+                       "   %soptions%s: %s"
                    , kq, kq
                    , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
                    , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
@@ -86,7 +87,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                    , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st))
                    , kq, kq, (uint32_t)r->before
                    , kq, kq, (uint32_t)r->after
-                   , kq, kq);
+                   , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
+                   , kq, kq, sq);
+
+    web_client_api_request_v1_data_options_to_string(wb, options);
+
+    buffer_sprintf(wb, "%s,\n   %sdimension_names%s: [", sq, kq, kq);
+
     if (should_lock)
         rrdset_unlock(r->st);
 
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index 65dbd5b658..58892d8892 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -4,9 +4,11 @@
 #define NETDATA_API_FORMATTER_JSON_WRAPPER_H
 
 #include "rrd2json.h"
+#include "web/api/queries/query.h"
+
 
 extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    QUERY_PARAMS *query_params);
+    RRDR_GROUPING group_method, QUERY_PARAMS *query_params);
 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 1de6be4e33..0393e5f2e8 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -162,6 +162,8 @@ int rrdset2value_api_v1(
         , uint32_t options
         , time_t *db_after
         , time_t *db_before
+        , size_t *db_points_read
+        , size_t *result_points_generated
         , int *value_is_null
         , int timeout
 ) {
@@ -177,9 +179,13 @@ int rrdset2value_api_v1(
         goto cleanup;
     }
 
-    if(rrdr_rows(r) == 0) {
-        rrdr_free(owa, r);
+    if(db_points_read)
+        *db_points_read += r->internal.db_points_read;
 
+    if(result_points_generated)
+        *result_points_generated += r->internal.result_points_generated;
+
+    if(rrdr_rows(r) == 0) {
         if(db_after)  *db_after  = 0;
         if(db_before) *db_before = 0;
         if(value_is_null) *value_is_null = 1;
@@ -266,7 +272,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -279,7 +285,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -292,7 +298,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
             rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
@@ -305,7 +311,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -318,7 +324,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -331,7 +337,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
             buffer_strcat(wb, "[\n");
             rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
@@ -348,7 +354,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -361,7 +367,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
             rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -379,7 +385,7 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
@@ -391,7 +397,7 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
@@ -402,7 +408,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
@@ -415,7 +421,7 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index 60bed5b903..3fae314317 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -92,6 +92,8 @@ extern int rrdset2value_api_v1(
         , uint32_t options
         , time_t *db_after
         , time_t *db_before
+        , size_t *db_points_read
+        , size_t *result_points_generated
         , int *value_is_null
         , int timeout
 );
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 97427d3237..120249ec6f 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -319,7 +319,8 @@
                   "match-ids",
                   "match-names",
                   "showcustomvars",
-                  "allow_past"
+                  "allow_past",
+                  "anomaly-bit"
                 ]
               },
               "default": [
@@ -484,7 +485,8 @@
                   "absolute-sum",
                   "null2zero",
                   "percentage",
-                  "unaligned"
+                  "unaligned",
+                  "anomaly-bit"
                 ]
               },
               "default": [
@@ -908,7 +910,7 @@
     "/alarms_values": {
       "get": {
         "summary": "Get a list of active or raised alarms on the server",
-        "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value`, `last_updated` time and alarm `status`.",
+        "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value`, `last_updated` time, and alarm `status`.",
         "parameters": [
           {
             "name": "all",
@@ -1129,6 +1131,154 @@
           }
         }
       }
+    },
+    "/metric_correlations": {
+      "get": {
+        "summary": "Analyze all the metrics to find their correlations",
+        "description": "Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
+        "parameters": [
+          {
+            "name": "baseline_after",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the starting point of baseline window, or a relative number of seconds (negative, relative to parameter baseline_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -300
+            }
+          },
+          {
+            "name": "baseline_before",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the ending point of the baseline window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
+            "required": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -60
+            }
+          },
+          {
+            "name": "highlight_after",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -60
+            }
+          },
+          {
+            "name": "highlight_before",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
+            "required": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 0
+            }
+          },
+          {
+            "name": "points",
+            "in": "query",
+            "description": "The number of points to be evaluated for the highlighted window. The baseline window will be adjusted automatically to receive a proportional amount of points.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 500
+            }
+          },
+          {
+            "name": "method",
+            "in": "query",
+            "description": "the algorithm to run",
+            "required": false,
+            "schema": {
+              "type": "string",
+              "enum": [
+                "ks2",
+                "volume"
+              ],
+              "default": "volume"
+            }
+          },
+          {
+            "name": "timeout",
+            "in": "query",
+            "description": "Cancel the query if to takes more that this amount of milliseconds.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 60000
+            }
+          },
+          {
+            "name": "options",
+            "in": "query",
+            "description": "Options that affect data generation.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [
+                  "min2max",
+                  "abs",
+                  "absolute",
+                  "absolute-sum",
+                  "null2zero",
+                  "percentage",
+                  "unaligned",
+                  "allow_past",
+                  "nonzero",
+                  "anomaly-bit",
+                  "raw"
+                ]
+              },
+              "default": [
+                "null2zero",
+                "allow_past",
+                "nonzero",
+                "unaligned"
+              ]
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "JSON object with weights for each chart and dimension.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/metric_correlations"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "The given parameters are invalid."
+          },
+          "403": {
+            "description": "metrics correlations are not enabled on this Netdata Agent."
+          },
+          "404": {
+            "description": "No charts could be found, or the method that correlated the metrics did not produce any result."
+          },
+          "504": {
+            "description": "Timeout - the query took too long and has been cancelled."
+          }
+        }
+      }
     }
   },
   "servers": [
@@ -1267,7 +1417,7 @@
           "stream_compression": {
             "type": "boolean",
             "description": "Stream transmission compression method.",
-            "example": "true"
+            "example": true
           },
           "labels": {
             "type": "object",
@@ -2135,7 +2285,7 @@
         "type": "object",
         "properties": {
           "aclk-available": {
-            "type": "boolean",
+            "type": "string",
             "description": "Describes whether this agent is capable of connection to the Cloud. False means agent has been built without ACLK component either on purpose (user choice) or due to missing dependency."
           },
           "aclk-version": {
@@ -2153,7 +2303,7 @@
             "type": "boolean",
             "description": "Informs whether this agent has been added to a space in the cloud (User has to perform claiming). If false (user didn't perform claiming) agent will never attempt any cloud connection."
           },
-          "claimed-id": {
+          "claimed_id": {
             "type": "string",
             "format": "uuid",
             "description": "Unique ID this agent uses to identify when connecting to cloud"
@@ -2171,7 +2321,60 @@
             ]
           }
         }
+      },
+      "metric_correlations": {
+        "type": "object",
+        "properties": {
+          "correlated_charts": {
+            "type": "object",
+            "description": "An object containing chart objects with their metrics correlations.",
+            "properties": {
+              "chart-id1": {
+                "type": "object",
+                "properties": {
+                  "context": {
+                    "type": "string"
+                  },
+                  "dimensions": {
+                    "type": "object",
+                    "properties": {
+                      "dimension1-name": {
+                        "type": "number"
+                      },
+                      "dimension2-name": {
+                        "type": "number"
+                      }
+                    }
+                  }
+                }
+              },
+              "chart-id2": {
+                "type": "object",
+                "properties": {
+                  "context": {
+                    "type": "string"
+                  },
+                  "dimensions": {
+                    "type": "object",
+                    "properties": {
+                      "dimension1-name": {
+                        "type": "number"
+                      },
+                      "dimension2-name": {
+                        "type": "number"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "total_dimensions_count": {
+            "description": "The number of dimensions correlated",
+            "type": "integer"
+          }
+        }
       }
     }
   }
-}
+}
\ No newline at end of file
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index 96920375e1..7049370508 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -280,6 +280,7 @@ paths:
                 - match-names
                 - showcustomvars
                 - allow_past
+                - anomaly-bit
             default:
               - seconds
               - jsonwrap
@@ -427,6 +428,7 @@ paths:
                 - null2zero
                 - percentage
                 - unaligned
+                - anomaly-bit
             default:
               - absolute
         - name: label
@@ -913,6 +915,138 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/aclk_state"
+  /metric_correlations:
+    get:
+      summary: Analyze all the metrics to find their correlations
+      description: Given two time-windows (baseline, highlight), it goes
+        through all the available metrics, querying both windows and tries to find
+        how these two windows relate to each other. It supports
+        multiple algorithms to do so. The result is a list of all
+        metrics evaluated, weighted for 0.0 (the two windows are
+        more different) to 1.0 (the two windows are similar).
+        The algorithm adjusts automatically the baseline window to be
+        a power of two multiple of the highlighted (1, 2, 4, 8, etc).
+      parameters:
+        - name: baseline_after
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            starting point of baseline window, or a relative number of
+            seconds (negative, relative to parameter baseline_before). Netdata will
+            assume it is a relative number if it is less that 3 years (in seconds).
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: -300
+        - name: baseline_before
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            ending point of the baseline window, or a relative number of
+            seconds (negative), relative to the last collected timestamp.
+            Netdata will assume it is a relative number if it is less than 3
+            years (in seconds).
+          required: false
+          schema:
+            type: number
+            format: integer
+            default: -60
+        - name: highlight_after
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            starting point of highlighted window, or a relative number of
+            seconds (negative, relative to parameter highlight_before). Netdata will
+            assume it is a relative number if it is less that 3 years (in seconds).
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: -60
+        - name: highlight_before
+          in: query
+          description: This parameter can either be an absolute timestamp specifying the
+            ending point of the highlighted window, or a relative number of
+            seconds (negative), relative to the last collected timestamp.
+            Netdata will assume it is a relative number if it is less than 3
+            years (in seconds).
+          required: false
+          schema:
+            type: number
+            format: integer
+            default: 0
+        - name: points
+          in: query
+          description: The number of points to be evaluated for the highlighted window.
+            The baseline window will be adjusted automatically to receive a proportional
+            amount of points.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: 500
+        - name: method
+          in: query
+          description: the algorithm to run
+          required: false
+          schema:
+            type: string
+            enum:
+              - ks2
+              - volume
+            default: volume
+        - name: timeout
+          in: query
+          description: Cancel the query if to takes more that this amount of milliseconds.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: 60000
+        - name: options
+          in: query
+          description: Options that affect data generation.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: array
+            items:
+              type: string
+              enum:
+                - min2max
+                - abs
+                - absolute
+                - absolute-sum
+                - null2zero
+                - percentage
+                - unaligned
+                - allow_past
+                - nonzero
+                - anomaly-bit
+                - raw
+            default:
+              - null2zero
+              - allow_past
+              - nonzero
+              - unaligned
+      responses:
+        "200":
+          description: JSON object with weights for each chart and dimension.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/metric_correlations"
+        "400":
+          description: The given parameters are invalid.
+        "403":
+          description: metrics correlations are not enabled on this Netdata Agent.
+        "404":
+          description: No charts could be found, or the method
+            that correlated the metrics did not produce any result.
+        "504":
+          description: Timeout - the query took too long and has been cancelled.
 servers:
   - url: https://registry.my-netdata.io/api/v1
   - url: http://registry.my-netdata.io/api/v1
@@ -1696,3 +1830,37 @@ components:
           enum:
             - Old
             - New
+    metric_correlations:
+      type: object
+      properties:
+        correlated_charts:
+          type: object
+          description: An object containing chart objects with their metrics correlations.
+          properties:
+            chart-id1:
+              type: object
+              properties:
+                context:
+                  type: string
+                dimensions:
+                  type: object
+                  properties:
+                    dimension1-name:
+                      type: number
+                    dimension2-name:
+                      type: number
+            chart-id2:
+              type: object
+              properties:
+                context:
+                  type: string
+                dimensions:
+                  type: object
+                  properties:
+                    dimension1-name:
+                      type: number
+                    dimension2-name:
+                      type: number
+        total_dimensions_count:
+          description: The number of dimensions correlated
+          type: integer
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 5c6c704119..5f46a50754 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -280,6 +280,16 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI
     return def;
 }
 
+const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group) {
+    int i;
+
+    for(i = 0; api_v1_data_groups[i].name ; i++)
+        if(unlikely(group == api_v1_data_groups[i].value))
+            return api_v1_data_groups[i].name;
+
+    return "unknown";
+}
+
 // ----------------------------------------------------------------------------
 
 static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
@@ -791,49 +801,36 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
 #endif // NETDATA_INTERNAL_CHECKS
 
 // Returns 1 if an absolute period was requested or 0 if it was a relative period
-static int rrdr_convert_before_after_to_absolute(
-        long long *after_requestedp
-        , long long *before_requestedp
-        , int update_every
-        , time_t first_entry_t
-        , time_t last_entry_t
-        , RRDR_OPTIONS options
-) {
+int rrdr_relative_window_to_absolute(long long *after, long long *before, int update_every, long points) {
+    time_t now = now_realtime_sec() - 1;
+
     int absolute_period_requested = -1;
     long long after_requested, before_requested;
 
-    before_requested = *before_requestedp;
-    after_requested = *after_requestedp;
-
-    if(before_requested == 0 && after_requested == 0) {
-        // dump the all the data
-        before_requested = last_entry_t;
-        after_requested = first_entry_t;
-        absolute_period_requested = 0;
-