Trimmed-median, trimmed-mean and percentile (#13469)

19 files changed, 1542 insertions, 47 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ef3a297d1..9752f37187 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -814,12 +814,16 @@ set(API_PLUGIN_FILES
         web/api/queries/sum/sum.h
         web/api/queries/median/median.c
         web/api/queries/median/median.h
+        web/api/queries/percentile/percentile.c
+        web/api/queries/percentile/percentile.h
         web/api/queries/stddev/stddev.c
         web/api/queries/stddev/stddev.h
         web/api/queries/ses/ses.c
         web/api/queries/ses/ses.h
         web/api/queries/des/des.c
         web/api/queries/des/des.h
+        web/api/queries/trimmed_mean/trimmed_mean.c
+        web/api/queries/trimmed_mean/trimmed_mean.h
         web/api/queries/weights.c
         web/api/queries/weights.h
         web/api/formatters/rrd2json.c
diff --git a/Makefile.am b/Makefile.am
index b962bb7973..7f8adf6afd 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -587,6 +587,10 @@ API_PLUGIN_FILES = \
     web/api/queries/median/median.h \
     web/api/queries/min/min.c \
     web/api/queries/min/min.h \
+    web/api/queries/percentile/percentile.c \
+    web/api/queries/percentile/percentile.h \
+    web/api/queries/trimmed_mean/trimmed_mean.c \
+    web/api/queries/trimmed_mean/trimmed_mean.h \
     web/api/queries/query.c \
     web/api/queries/query.h \
     web/api/queries/rrdr.c \
diff --git a/configure.ac b/configure.ac
index 9dbb6a7c1a..40cd3869da 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1748,9 +1748,11 @@ AC_CONFIG_FILES([
     web/api/queries/max/Makefile
     web/api/queries/median/Makefile
     web/api/queries/min/Makefile
+    web/api/queries/percentile/Makefile
     web/api/queries/ses/Makefile
     web/api/queries/stddev/Makefile
     web/api/queries/sum/Makefile
+    web/api/queries/trimmed_mean/Makefile
     web/api/health/Makefile
     web/gui/Makefile
     web/gui/dashboard/Makefile
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 5fc110f27b..029783b55b 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -453,7 +453,35 @@
                 "ses",
                 "des",
                 "cv",
-                "countif"
+                "countif",
+                "percentile",
+                "percentile25",
+                "percentile50",
+                "percentile75",
+                "percentile80",
+                "percentile90",
+                "percentile95",
+                "percentile97",
+                "percentile98",
+                "percentile99",
+                "trimmed-mean",
+                "trimmed-mean1",
+                "trimmed-mean2",
+                "trimmed-mean3",
+                "trimmed-mean5",
+                "trimmed-mean10",
+                "trimmed-mean15",
+                "trimmed-mean20",
+                "trimmed-mean25",
+                "trimmed-median",
+                "trimmed-median1",
+                "trimmed-median2",
+                "trimmed-median3",
+                "trimmed-median5",
+                "trimmed-median10",
+                "trimmed-median15",
+                "trimmed-median20",
+                "trimmed-median25"
               ],
               "default": "average"
             }
@@ -690,7 +718,39 @@
                 "median",
                 "stddev",
                 "sum",
-                "incremental-sum"
+                "incremental-sum",
+                "ses",
+                "des",
+                "cv",
+                "countif",
+                "percentile",
+                "percentile25",
+                "percentile50",
+                "percentile75",
+                "percentile80",
+                "percentile90",
+                "percentile95",
+                "percentile97",
+                "percentile98",
+                "percentile99",
+                "trimmed-mean",
+                "trimmed-mean1",
+                "trimmed-mean2",
+                "trimmed-mean3",
+                "trimmed-mean5",
+                "trimmed-mean10",
+                "trimmed-mean15",
+                "trimmed-mean20",
+                "trimmed-mean25",
+                "trimmed-median",
+                "trimmed-median1",
+                "trimmed-median2",
+                "trimmed-median3",
+                "trimmed-median5",
+                "trimmed-median10",
+                "trimmed-median15",
+                "trimmed-median20",
+                "trimmed-median25"
               ],
               "default": "average"
             }
@@ -1362,7 +1422,7 @@
     "/metric_correlations": {
       "get": {
         "summary": "Analyze all the metrics to find their correlations",
-        "description": "Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
+        "description": "THIS ENDPOINT IS OBSOLETE. Use the /weights endpoint. Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
         "parameters": [
           {
             "name": "baseline_after",
@@ -1499,7 +1559,35 @@
                 "ses",
                 "des",
                 "cv",
-                "countif"
+                "countif",
+                "percentile",
+                "percentile25",
+                "percentile50",
+                "percentile75",
+                "percentile80",
+                "percentile90",
+                "percentile95",
+                "percentile97",
+                "percentile98",
+                "percentile99",
+                "trimmed-mean",
+                "trimmed-mean1",
+                "trimmed-mean2",
+                "trimmed-mean3",
+                "trimmed-mean5",
+                "trimmed-mean10",
+                "trimmed-mean15",
+                "trimmed-mean20",
+                "trimmed-mean25",
+                "trimmed-median",
+                "trimmed-median1",
+                "trimmed-median2",
+                "trimmed-median3",
+                "trimmed-median5",
+                "trimmed-median10",
+                "trimmed-median15",
+                "trimmed-median20",
+                "trimmed-median25"
               ],
               "default": "average"
             }
@@ -1540,6 +1628,236 @@
           }
         }
       }
+    },
+    "/weights": {
+      "get": {
+        "summary": "Analyze all the metrics using an algorithm and score them accordingly",
+        "description": "This endpoint goes through all metrics and scores them according to an algorithm.",
+        "parameters": [
+          {
+            "name": "baseline_after",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the starting point of baseline window, or a relative number of seconds (negative, relative to parameter baseline_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -300
+            }
+          },
+          {
+            "name": "baseline_before",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the ending point of the baseline window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
+            "required": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -60
+            }
+          },
+          {
+            "name": "after",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": -60
+            }
+          },
+          {
+            "name": "before",
+            "in": "query",
+            "description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
+            "required": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 0
+            }
+          },
+          {
+            "name": "context",
+            "in": "query",
+            "description": "A simple pattern matching the contexts to evaluate.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "points",
+            "in": "query",
+            "description": "The number of points to be evaluated for the highlighted window. The baseline window will be adjusted automatically to receive a proportional amount of points. This parameter is only used by the KS2 algorithm.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 500
+            }
+          },
+          {
+            "name": "method",
+            "in": "query",
+            "description": "the algorithm to run",
+            "required": false,
+            "schema": {
+              "type": "string",
+              "enum": [
+                "ks2",
+                "volume",
+                "anomaly-rate"
+              ],
+              "default": "anomaly-rate"
+            }
+          },
+          {
+            "name": "tier",
+            "in": "query",
+            "description": "Use the specified database tier",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer"
+            }
+          },
+          {
+            "name": "timeout",
+            "in": "query",
+            "description": "Cancel the query if to takes more that this amount of milliseconds.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 60000
+            }
+          },
+          {
+            "name": "options",
+            "in": "query",
+            "description": "Options that affect data generation.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [
+                  "min2max",
+                  "abs",
+                  "absolute",
+                  "absolute-sum",
+                  "null2zero",
+                  "percentage",
+                  "unaligned",
+                  "nonzero",
+                  "anomaly-bit",
+                  "raw"
+                ]
+              },
+              "default": [
+                "null2zero",
+                "nonzero",
+                "unaligned"
+              ]
+            }
+          },
+          {
+            "name": "group",
+            "in": "query",
+            "description": "The grouping method. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported \"min\", \"max\", \"average\", \"sum\", \"incremental-sum\". \"max\" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).",
+            "required": true,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "string",
+              "enum": [
+                "min",
+                "max",
+                "average",
+                "median",
+                "stddev",
+                "sum",
+                "incremental-sum",
+                "ses",
+                "des",
+                "cv",
+                "countif",
+                "percentile",
+                "percentile25",
+                "percentile50",
+                "percentile75",
+                "percentile80",
+                "percentile90",
+                "percentile95",
+                "percentile97",
+                "percentile98",
+                "percentile99",
+                "trimmed-mean",
+                "trimmed-mean1",
+                "trimmed-mean2",
+                "trimmed-mean3",
+                "trimmed-mean5",
+                "trimmed-mean10",
+                "trimmed-mean15",
+                "trimmed-mean20",
+                "trimmed-mean25",
+                "trimmed-median",
+                "trimmed-median1",
+                "trimmed-median2",
+                "trimmed-median3",
+                "trimmed-median5",
+                "trimmed-median10",
+                "trimmed-median15",
+                "trimmed-median20",
+                "trimmed-median25"
+              ],
+              "default": "average"
+            }
+          },
+          {
+            "name": "group_options",
+            "in": "query",
+            "description": "When the group function supports additional parameters, this field can be used to pass them to it. Currently only \"countif\" supports this.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "JSON object with weights for each context, chart and dimension.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/weights"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "The given parameters are invalid."
+          },
+          "403": {
+            "description": "metrics correlations are not enabled on this Netdata Agent."
+          },
+          "404": {
+            "description": "No charts could be found, or the method that correlated the metrics did not produce any result."
+          },
+          "504": {
+            "description": "Timeout - the query took too long and has been cancelled."
+          }
+        }
+      }
     }
   },
   "servers": [
@@ -2785,6 +3103,124 @@
             }
           }
         }
+      },
+      "weights": {
+        "type": "object",
+        "properties": {
+          "after": {
+            "description": "the start time of the highlighted window",
+            "type": "integer"
+          },
+          "before": {
+            "description": "the end time of the highlighted window",
+            "type": "integer"
+          },
+          "duration": {
+            "description": "the duration of the highlighted window",
+            "type": "integer"
+          },
+          "points": {
+            "description": "the points of the highlighted window",
+            "type": "integer"
+          },
+          "baseline_after": {
+            "description": "the start time of the baseline window",
+            "type": "integer"
+          },
+          "baseline_before": {
+            "description": "the end time of the baseline window",
+            "type": "integer"
+          },
+          "baseline_duration": {
+            "description": "the duration of the baseline window",
+            "type": "integer"
+          },
+          "baseline_points": {
+            "description": "the points of the baseline window",
+            "type": "integer"
+          },
+          "group": {
+            "description": "the grouping method across time",
+            "type": "string"
+          },
+          "method": {
+            "description": "the correlation method used",
+            "type": "string"
+          },
+          "options": {
+            "description": "a comma separated list of the query options set",
+            "type": "string"
+          },
+          "correlated_dimensions": {
+            "description": "the number of dimensions returned in the result"
+          },
+          "total_dimensions_count": {
+            "description": "the total number of dimensions evaluated",
+            "type": "integer"
+          },
+          "statistics": {
+            "type": "object",
+            "properties": {
+              "query_time_ms": {
+                "type": "number"
+              },
+              "db_queries": {
+                "type": "integer"
+              },
+              "db_points_read": {
+                "type": "integer"
+              },
+              "query_result_points": {
+                "type": "integer"
+              },
+              "binary_searches": {
+                "type": "integer"
+              }
+            }
+          },
+          "contexts": {
+            "description": "A dictionary of weighted context objects.",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/components/schemas/weighted_context"
+            }
+          }
+        }
+      },
+      "weighted_context": {
+        "type": "object",
+        "properties": {
+          "weight": {
+            "description": "The average weight of the context.",
+            "type": "number"
+          },
+          "charts": {
+            "description": "A dictionary of weighted chart objects.",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/components/schemas/weighted_chart"
+            }
+          }
+        }
+      },
+      "weighted_chart": {
+        "type": "object",
+        "properties": {
+          "weight": {
+            "description": "The average weight of the context.",
+            "type": "number"
+          },
+          "dimensions": {
+            "description": "A dictionary of weighted dimensions.",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/components/schemas/weighted_dimension"
+            }
+          }
+        }
+      },
+      "weighted_dimension": {
+        "type": "number"
       }
     }
   }
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index c6282c13c8..2e04e9f20f 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -365,6 +365,34 @@ paths:
               - des
               - cv
               - countif
+              - percentile
+              - percentile25
+              - percentile50
+              - percentile75
+              - percentile80
+              - percentile90
+              - percentile95
+              - percentile97
+              - percentile98
+              - percentile99
+              - trimmed-mean
+              - trimmed-mean1
+              - trimmed-mean2
+              - trimmed-mean3
+              - trimmed-mean5
+              - trimmed-mean10
+              - trimmed-mean15
+              - trimmed-mean20
+              - trimmed-mean25
+              - trimmed-median
+              - trimmed-median1
+              - trimmed-median2
+              - trimmed-median3
+              - trimmed-median5
+              - trimmed-median10
+              - trimmed-median15
+              - trimmed-median20
+              - trimmed-median25
             default: average
         - name: group_options
           in: query
@@ -575,6 +603,38 @@ paths:
               - stddev
               - sum
               - incremental-sum
+              - ses
+              - des
+              - cv
+              - countif
+              - percentile
+              - percentile25
+              - percentile50
+              - percentile75
+              - percentile80
+              - percentile90
+              - percentile95
+              - percentile97
+              - percentile98
+              - percentile99
+              - trimmed-mean
+              - trimmed-mean1
+              - trimmed-mean2
+              - trimmed-mean3
+              - trimmed-mean5
+              - trimmed-mean10
+              - trimmed-mean15
+              - trimmed-mean20
+              - trimmed-mean25
+              - trimmed-median
+              - trimmed-median1
+              - trimmed-median2
+              - trimmed-median3
+              - trimmed-median5
+              - trimmed-median10
+              - trimmed-median15
+              - trimmed-median20
+              - trimmed-median25
             default: average
         - name: options
           in: query
@@ -1238,6 +1298,34 @@ paths:
               - des
               - cv
               - countif
+              - percentile
+              - percentile25
+              - percentile50
+              - percentile75
+              - percentile80
+              - percentile90
+              - percentile95
+              - percentile97
+              - percentile98
+              - percentile99
+              - trimmed-mean
+              - trimmed-mean1
+              - trimmed-mean2
+              - trimmed-mean3
+              - trimmed-mean5
+              - trimmed-mean10
+              - trimmed-mean15
+              - trimmed-mean20
+              - trimmed-mean25
+              - trimmed-median
+              - trimmed-median1
+              - trimmed-median2
+              - trimmed-median3
+              - trimmed-median5
+              - trimmed-median10
+              - trimmed-median15
+              - trimmed-median20
+              - trimmed-median25
             default: average
         - name: group_options
           in: query
@@ -1413,6 +1501,34 @@ paths:
               - des
               - cv
               - countif
+              - percentile
+              - percentile25
+              - percentile50
+              - percentile75
+              - percentile80
+              - percentile90
+              - percentile95
+              - percentile97
+              - percentile98
+              - percentile99
+              - trimmed-mean
+              - trimmed-mean1
+              - trimmed-mean2
+              - trimmed-mean3
+              - trimmed-mean5
+              - trimmed-mean10
+              - trimmed-mean15
+              - trimmed-mean20
+              - trimmed-mean25
+              - trimmed-median
+              - trimmed-median1
+              - trimmed-median2
+              - trimmed-median3
+              - trimmed-median5
+              - trimmed-median10
+              - trimmed-median15
+              - trimmed-median20
+              - trimmed-median25
             default: average
         - name: group_options
           in: query
@@ -1428,7 +1544,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/weight"
+                $ref: "#/components/schemas/weights"
         "400":
           description: The given parameters are invalid.
         "403":
@@ -2373,7 +2489,7 @@ components:
                       type: number
                     dimension2-name:
                       type: number
-    weight:
+    weights:
       type: object
       properties:
         after:
@@ -2428,26 +2544,31 @@ components:
             binary_searches:
               type: integer
         contexts:
+          description: A dictionary of weighted context objects.
           type: object
-          description: An object containing context objects.
-          properties:
-            contextX:
-              type: object
-              properties:
-                charts:
-                  type: object
-                  properties:
-                    chartX:
-                      type: object
-                      properties:
-                        dimensions:
-                          type: object
-                          properties:
-                            dimensionX:
-                              type: number
-                        weight:
-                          description: The average chart weight
-                          type: number
-                      weight:
-                        description: The average context weight
-                        type: number
+          additionalProperties:
+            $ref: '#/components/schemas/weighted_context'
+    weighted_context:
+      type: object
+      properties:
+        weight:
+          description: The average weight of the context.
+          type: number
+        charts:
+          description: A dictionary of weighted chart objects.
+          type: object
+          additionalProperties:
+            $ref: '#/components/schemas/weighted_chart'
+    weighted_chart:
+      type: object
+      properties:
+        weight:
+          description: The average weight of the context.
+          type: number
+        dimensions:
+          description: A dictionary of weighted dimensions.
+          type: object
+          additionalProperties:
+            $ref: '#/components/schemas/weighted_dimension'
+    weighted_dimension:
+      type: number
diff --git a/web/api/queries/Makefile.am b/web/api/queries/Makefile.am
index 89fe15911b..7c4c435205 100644
--- a/web/api/queries/Makefile.am
+++ b/web/api/queries/Makefile.am
@@ -12,8 +12,10 @@ SUBDIRS = \
     min \
     sum \
     median \
+    percentile \
     ses \
     stddev \
+    trimmed_mean \
     $(NULL)
 
 dist_noinst_DATA = \
diff --git a/web/api/queries/median/README.md b/web/api/queries/median/README.md
index bb7d4c66be..5600284c22 100644
--- a/web/api/queries/median/README.md
+++ b/web/api/queries/median/README.md
@@ -13,6 +13,20 @@ The median is the value separating the higher half from the lower half of a data
 `median` is not an accurate average. However, it eliminates all spikes, by sorting
 all the values in a period, and selecting the value in the middle of the sorted array.
 
+Netdata also supports `trimmed-median`, which trims a percentage of the smaller and bigger values prior to finding the
+median. The following `trimmed-median` functions are defined:
+
+- `trimmed-median1`
+- `trimmed-median2`
+- `trimmed-median3`
+- `trimmed-median5`
+- `trimmed-median10`
+- `trimmed-median15`
+- `trimmed-median20`
+- `trimmed-median25`
+
+The function `trimmed-median` is an alias for `trimmed-median5`.
+
 ## how to use
 
 Use it in alarms like this:
@@ -27,7 +41,8 @@ lookup: median -1m unaligned of my_dimension
 `median` does not change the units. For example, if the chart units is `requests/sec`, the result
 will be again expressed in the same units. 
 
-It can also be used in APIs and badges as `&group=median` in the URL.
+It can also be used in APIs and badges as `&group=median` in the URL. Additionally, a percentage may be given with
+`&group_options=` to trim all small and big values before finding the median.
 
 ## Examples
 
diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c
index d5fa1e5b6b..40fd4ec3af 100644
--- a/web/api/queries/median/median.c
+++ b/web/api/queries/median/median.c
@@ -2,28 +2,65 @@
 
 #include "median.h"
 
-
 // ----------------------------------------------------------------------------
 // median
 
 struct grouping_median {
     size_t series_size;
     size_t next_pos;
+    NETDATA_DOUBLE percent;
 
     NETDATA_DOUBLE *series;
 };
 
-void grouping_create_median(RRDR *r, const char *options __maybe_unused) {
+void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
     long entries = r->group;
-    if(entries < 0) entries = 0;
+    if(entries < 10) entries = 10;
 
     struct grouping_median *g = (struct groupi