DYNCFG: alerts improvements (#17165)

* updated schema

* move "match" inside "config" in the json representation of the alert

* removed green and red from everywhere, but if they exist in the config file they are replaced with fixed numeric values

* additional health fields

* default permissions for systemd dyncfg; remove test from alerts

* added the ability to define time grouping options

* updated prototype

* Add new fields to the database and store

* Read newly stored values when fetching config (not exposed to JSON)

* Render new values

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

29 files changed, 1108 insertions, 527 deletions

diff --git a/src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c b/src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
index 490d4b795a..469f9d2cf8 100644
--- a/src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
+++ b/src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
@@ -156,8 +156,8 @@ void systemd_journal_dyncfg_init(struct functions_evloop_globals *wg) {
         DYNCFG_SOURCE_TYPE_INTERNAL,
         "internal",
         DYNCFG_CMD_SCHEMA | DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE,
-        HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_VIEW_AGENT_CONFIG,
-        HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_EDIT_AGENT_CONFIG,
+        HTTP_ACCESS_NONE,
+        HTTP_ACCESS_NONE,
         systemd_journal_directories_dyncfg_cb,
         NULL);
 }
diff --git a/src/daemon/config/dyncfg.c b/src/daemon/config/dyncfg.c
index 5ee0dcfb60..2a5696b2ac 100644
--- a/src/daemon/config/dyncfg.c
+++ b/src/daemon/config/dyncfg.c
@@ -291,7 +291,7 @@ bool dyncfg_add_low_level(RRDHOST *host, const char *id, const char *path,
         view_access = HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_VIEW_AGENT_CONFIG;
 
     if(edit_access == HTTP_ACCESS_NONE)
-        edit_access = HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_EDIT_AGENT_CONFIG;
+        edit_access = HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_EDIT_AGENT_CONFIG | HTTP_ACCESS_COMMERCIAL_SPACE;
 
     if(!dyncfg_is_valid_id(id)) {
         nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
diff --git a/src/database/contexts/api_v2.c b/src/database/contexts/api_v2.c
index 2f6b1ff1ca..24163e703c 100644
--- a/src/database/contexts/api_v2.c
+++ b/src/database/contexts/api_v2.c
@@ -1338,6 +1338,10 @@ static void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct s
 
                     buffer_json_member_add_time_t(wb, "after", t->value.db.after);
                     buffer_json_member_add_time_t(wb, "before", t->value.db.before);
+                    buffer_json_member_add_string(wb, "time_group_condition", alerts_group_conditions_id2txt(t->value.db.time_group_condition));
+                    buffer_json_member_add_double(wb, "time_group_value", t->value.db.time_group_value);
+                    buffer_json_member_add_string(wb, "dims_group", alerts_dims_grouping_id2group(t->value.db.dims_group));
+                    buffer_json_member_add_string(wb, "data_source", alerts_data_source_id2source(t->value.db.data_source));
                     buffer_json_member_add_string(wb, "method", t->value.db.method);
                     buffer_json_member_add_string(wb, "dimensions", t->value.db.dimensions);
                     rrdr_options_to_buffer_json_array(wb, "options", (RRDR_OPTIONS)t->value.db.options);
diff --git a/src/database/contexts/rrdcontext.h b/src/database/contexts/rrdcontext.h
index bdac3acddd..08a5760b55 100644
--- a/src/database/contexts/rrdcontext.h
+++ b/src/database/contexts/rrdcontext.h
@@ -476,6 +476,10 @@ struct sql_alert_config_data {
         struct {
             const char *dimensions;
             const char *method;
+            ALERT_LOOKUP_TIME_GROUP_CONDITION time_group_condition;
+            NETDATA_DOUBLE time_group_value;
+            ALERT_LOOKUP_DIMS_GROUPING dims_group;
+            ALERT_LOOKUP_DATA_SOURCE data_source;
             uint32_t options;
 
             int32_t after;
diff --git a/src/database/sqlite/sqlite_aclk_alert.c b/src/database/sqlite/sqlite_aclk_alert.c
index ea2eb51c79..c96f0eef8e 100644
--- a/src/database/sqlite/sqlite_aclk_alert.c
+++ b/src/database/sqlite/sqlite_aclk_alert.c
@@ -610,7 +610,7 @@ void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_has
                 netdata_log_error("aclk_push_alert_config_event: Unexpected param number %d", param);
 
             BUFFER *tmp_buf = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
-            buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 28));
+            rrdr_options_to_buffer(tmp_buf, sqlite3_column_int(res, 28));
             alarm_config.p_db_lookup_options = strdupz((char *)buffer_tostring(tmp_buf));
             buffer_free(tmp_buf);
 
diff --git a/src/database/sqlite/sqlite_db_migration.c b/src/database/sqlite/sqlite_db_migration.c
index 977c834489..6fd7f6ad65 100644
--- a/src/database/sqlite/sqlite_db_migration.c
+++ b/src/database/sqlite/sqlite_db_migration.c
@@ -153,6 +153,15 @@ const char *database_migrate_v13_v14[] = {
     NULL
 };
 
+const char *database_migrate_v16_v17[] = {
+    "ALTER TABLE alert_hash ADD time_group_condition INT",
+    "ALTER TABLE alert_hash ADD time_group_value DOUBLE",
+    "ALTER TABLE alert_hash ADD dims_group INT",
+    "ALTER TABLE alert_hash ADD data_source INT",
+    NULL
+};
+
+
 static int do_migration_v1_v2(sqlite3 *database)
 {
     if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops"))
@@ -430,6 +439,15 @@ static int do_migration_v15_v16(sqlite3 *database)
     return 0;
 }
 
+static int do_migration_v16_v17(sqlite3 *database)
+{
+    if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "time_group_condition"))
+        return init_database_batch(database, &database_migrate_v16_v17[0], "meta_migrate");
+
+    return 0;
+}
+
+
 static int do_migration_v12_v13(sqlite3 *database)
 {
     int rc = 0;
@@ -527,6 +545,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {
     {.name = "v13 to v14",  .func = do_migration_v13_v14},
     {.name = "v14 to v15",  .func = do_migration_v14_v15},
     {.name = "v15 to v16",  .func = do_migration_v15_v16},
+    {.name = "v16 to v17",  .func = do_migration_v16_v17},
     // the terminator of this array
     {.name = NULL, .func = NULL}
 };
diff --git a/src/database/sqlite/sqlite_health.c b/src/database/sqlite/sqlite_health.c
index 45c61781a9..ea883c51b9 100644
--- a/src/database/sqlite/sqlite_health.c
+++ b/src/database/sqlite/sqlite_health.c
@@ -895,16 +895,19 @@ void sql_health_alarm_log_load(RRDHOST *host)
 /*
  * Store an alert config hash in the database
  */
-#define SQL_STORE_ALERT_CONFIG_HASH                                                                                     \
-    "insert or replace into alert_hash (hash_id, date_updated, alarm, template, "                                       \
-    "on_key, class, component, type, lookup, every, units, calc, "                                                      \
-    "green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, "                                 \
-    "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, "                              \
-    "p_db_lookup_before, p_update_every, source, chart_labels, summary) values (@hash_id,UNIXEPOCH(),@alarm,@template," \
-    "@on_key,@class,@component,@type,@lookup,@every,@units,@calc,"                                                      \
-    "@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels,"                                 \
-    "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after,"                              \
-    "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary)"
+#define SQL_STORE_ALERT_CONFIG_HASH                                                                                    \
+    "insert or replace into alert_hash (hash_id, date_updated, alarm, template, "                                      \
+    "on_key, class, component, type, lookup, every, units, calc, "                                                     \
+    "green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, "                                \
+    "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, "                             \
+    "p_db_lookup_before, p_update_every, source, chart_labels, summary, time_group_condition, "                        \
+    "time_group_value, dims_group, data_source) "                                                                      \
+    "values (@hash_id,UNIXEPOCH(),@alarm,@template,"                                                                   \
+    "@on_key,@class,@component,@type,@lookup,@every,@units,@calc,"                                                     \
+    "@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels,"                                \
+    "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after,"                             \
+    "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary, @time_group_condition, "                      \
+    "@time_group_value, @dims_group, @data_source)"
 
 int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
 {
@@ -966,7 +969,8 @@ int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.lookup, ++param);
+    // Rebuild lookup
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param); // lookup line
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
@@ -985,11 +989,13 @@ int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_double(res, ++param, ap->config.green);
+    NETDATA_DOUBLE green = NAN;
+    rc = sqlite3_bind_double(res, ++param, green);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_double(res, ++param, ap->config.red);
+    NETDATA_DOUBLE red = NAN;
+    rc = sqlite3_bind_double(res, ++param, red);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
@@ -1056,11 +1062,11 @@ int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
         if (unlikely(rc != SQLITE_OK))
             goto bind_fail;
 
-        rc = sqlite3_bind_text(res, ++param, time_grouping_id2txt(ap->config.group), -1, SQLITE_STATIC);
+        rc = sqlite3_bind_text(res, ++param, time_grouping_id2txt(ap->config.time_group), -1, SQLITE_STATIC);
         if (unlikely(rc != SQLITE_OK))
             goto bind_fail;
 
-        rc = sqlite3_bind_int(res, ++param, (int) ap->config.options);
+        rc = sqlite3_bind_int(res, ++param, (int) RRDR_OPTIONS_REMOVE_OVERLAPPING(ap->config.options));
         if (unlikely(rc != SQLITE_OK))
             goto bind_fail;
 
@@ -1109,6 +1115,22 @@ int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
+    rc = sqlite3_bind_int(res, ++param, ap->config.time_group_condition);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
+    rc = sqlite3_bind_double(res, ++param, ap->config.time_group_value);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
+    rc = sqlite3_bind_int(res, ++param, ap->config.dims_group);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
+    rc = sqlite3_bind_int(res, ++param, ap->config.data_source);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
     rc = execute_insert(res);
     if (unlikely(rc != SQLITE_DONE))
         error_report("Failed to store alert config, rc = %d", rc);
@@ -1838,7 +1860,8 @@ done_only_drop:
     "SELECT ah.hash_id, alarm, template, on_key, class, component, type, lookup, every, "                              \
     " units, calc, families, green, red, warn, crit, "                                                                 \
     " exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, "           \
-    " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary "      \
+    " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary,  "    \
+    " time_group_condition, time_group_value, dims_group, data_source "                                                \
     " FROM alert_hash ah, c_%p t where ah.hash_id = t.hash_id"
 
 int sql_get_alert_configuration(
@@ -1943,6 +1966,10 @@ int sql_get_alert_configuration(
         acd.source = (const char *) sqlite3_column_text(res, param++);
         acd.selectors.chart_labels = (const char *) sqlite3_column_text(res, param++);
         acd.summary = (const char *) sqlite3_column_text(res, param++);
+        acd.value.db.time_group_condition =(int32_t) sqlite3_column_int(res, param++);
+        acd.value.db.time_group_value = sqlite3_column_double(res, param++);
+        acd.value.db.dims_group = (int32_t) sqlite3_column_int(res, param++);
+        acd.value.db.data_source = (int32_t) sqlite3_column_int(res, param++);
 
         cb(&acd, data);
         added++;
diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c
index 6dcb8f86f8..274a51eef3 100644
--- a/src/database/sqlite/sqlite_metadata.c
+++ b/src/database/sqlite/sqlite_metadata.c
@@ -4,7 +4,7 @@
 #include "sqlite3recover.h"
 //#include "sqlite_db_migration.h"
 
-#define DB_METADATA_VERSION 16
+#define DB_METADATA_VERSION 17
 
 const char *database_config[] = {
     "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
diff --git a/src/health/REFERENCE.md b/src/health/REFERENCE.md
index 478fd9c9dc..85f1d22817 100644
--- a/src/health/REFERENCE.md
+++ b/src/health/REFERENCE.md
@@ -385,7 +385,7 @@ This line makes a database lookup to find a value. This result of this lookup is
 The format is:
 
 ```yaml
-lookup: METHOD AFTER [at BEFORE] [every DURATION] [OPTIONS] [of DIMENSIONS]
+lookup: METHOD(GROUPING OPTIONS) AFTER [at BEFORE] [every DURATION] [OPTIONS] [of DIMENSIONS]
 ```
 
 The full [database query API](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md) is supported. In short:
@@ -393,6 +393,8 @@ The full [database query API](https://github.com/netdata/netdata/blob/master/src
 - `METHOD` is one of  the available [grouping methods](https://github.com/netdata/netdata/blob/master/src/web/api/queries/README.md#grouping-methods) such as `average`, `min`, `max` etc.
      This is required.
 
+  - `GROUPING OPTIONS` are optional and can have the form `CONDITION VALUE`, where `CONDITION` is `!=`, `=`, `<=`, `<`, `>`, `>=` and `VALUE` is a number. The `CONDITION` and `VALUE` are required for `countif`, while `VALUE` is used by `percentile`, `trimmed_mean` and `trimmed_median`.
+
 - `AFTER` is a relative number of seconds, but it also accepts a single letter for changing
      the units, like `-1s` = 1 second in the past, `-1m` = 1 minute in the past, `-1h` = 1 hour
      in the past, `-1d` = 1 day in the past. You need a negative number (i.e. how far in the past
@@ -404,8 +406,19 @@ The full [database query API](https://github.com/netdata/netdata/blob/master/src
 - `every DURATION` sets the updated frequency of the lookup (supports single letter units as
      above too).
 
-- `OPTIONS` is a space separated list of `percentage`, `absolute`, `min2max`, `unaligned`,
-     `match-ids`, `match-names`. Check the [badges](https://github.com/netdata/netdata/blob/master/src/web/api/badges/README.md) documentation for more info.
+- `OPTIONS` is a space separated list of `percentage`, `absolute`, `min`, `max`, `average`, `sum`,
+     `min2max`, `unaligned`, `match-ids`, `match-names`.
+
+  - `percentage` during time-aggregation, calculate the percentage of the selected dimensions over the total of all dimensions.
+  - `absolute` during time-aggregation, turns all sample values positive before using them.
+  - `min` after time-aggregation of each dimension, return the minimum of all dimensions.
+  - `max` after time-aggregation of each dimension, return the maximum of all dimensions.
+  - `average` after time-aggregation of each dimension, return the average of all dimensions.
+  - `sum` after time-aggregation of each dimension, return the sum of all dimensions (this is the default).
+  - `min2max` after time-aggregation of each dimension, return the delta between the min and the max of the dimensions.
+  - `unligned` prevents shifting the query window to multiples of the query duration.
+  - `match-ids` matches the dimensions based on their IDs (the default is enabled, give `match-names` to disable).
+  - `match-names` matches the dimension based on their names (the default is enabled, give `match-ids` to disable).
 
 - `of DIMENSIONS` is optional and has to be the last parameter. Dimensions have to be separated
      by `,` or `|`. The space characters found in dimensions will be kept as-is (a few dimensions
diff --git a/src/health/health.h b/src/health/health.h
index 882309cae1..2a1dd6c656 100644
--- a/src/health/health.h
+++ b/src/health/health.h
@@ -20,6 +20,10 @@ typedef enum __attribute__((packed)) {
     HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION = 0x80000000,
 } HEALTH_ENTRY_FLAGS;
 
+#define RRDR_OPTIONS_DATA_SOURCES (RRDR_OPTION_PERCENTAGE|RRDR_OPTION_ANOMALY_BIT)
+#define RRDR_OPTIONS_DIMS_AGGREGATION (RRDR_OPTION_DIMS_MIN|RRDR_OPTION_DIMS_MAX|RRDR_OPTION_DIMS_AVERAGE|RRDR_OPTION_DIMS_MIN2MAX)
+#define RRDR_OPTIONS_REMOVE_OVERLAPPING(options) ((options) &= ~(RRDR_OPTIONS_DIMS_AGGREGATION|RRDR_OPTIONS_DATA_SOURCES))
+
 void health_entry_flags_to_json_array(BUFFER *wb, const char *key, HEALTH_ENTRY_FLAGS flags);
 
 #ifndef HEALTH_LISTEN_PORT
diff --git a/src/health/health_config.c b/src/health/health_config.c
index 948f263a0b..d8c735c3f6 100644
--- a/src/health/health_config.c
+++ b/src/health/health_config.c
@@ -162,25 +162,21 @@ static inline int isvariableterm(const char s) {
     return 1;
 }
 
-static inline int health_parse_db_lookup(
-        size_t line, const char *filename, char *string,
-        RRDR_TIME_GROUPING *group_method, int *after, int *before, int *every,
-        RRDR_OPTIONS *options, STRING **dimensions
-) {
-    netdata_log_debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s: %s", line, filename, string);
-
-    if(*dimensions) string_freez(*dimensions);
-    *dimensions = NULL;
-    *after = 0;
-    *before = 0;
-    *every = 0;
-    *options = 0;
+static inline int health_parse_db_lookup(size_t line, const char *filename, char *string, struct rrd_alert_config *ac) {
+    if(ac->dimensions) string_freez(ac->dimensions);
+    ac->dimensions = NULL;
+    ac->after = 0;
+    ac->before = 0;
+    ac->update_every = 0;
+    ac->options = 0;
+    ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_EQUAL;
+    ac->time_group_value = NAN;
 
     char *s = string, *key;
 
     // first is the group method
     key = s;
-    while(*s && !isspace(*s)) s++;
+    while(*s && !isspace(*s) && *s != '(') s++;
     while(*s && isspace(*s)) *s++ = '\0';
     if(!*s) {
         netdata_log_error("Health configuration invalid chart calculation at line %zu of file '%s': expected group method followed by the 'after' time, but got '%s'",
@@ -188,25 +184,103 @@ static inline int health_parse_db_lookup(
         return 0;
     }
 
-    if((*group_method = time_grouping_parse(key, RRDR_GROUPING_UNDEFINED)) == RRDR_GROUPING_UNDEFINED) {
+    bool group_options = false;
+    if(*s == '(') {
+        *s++ = '\0';
+        group_options = true;
+    }
+
+    if((ac->time_group = time_grouping_parse(key, RRDR_GROUPING_UNDEFINED)) == RRDR_GROUPING_UNDEFINED) {
         netdata_log_error("Health configuration at line %zu of file '%s': invalid group method '%s'",
                           line, filename, key);
         return 0;
     }
 
+    if(group_options) {
+        if(*s == '!') {
+            s++;
+            if(*s == '=') s++;
+            ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_NOT_EQUAL;
+        }
+        else if(*s == '<') {
+            s++;
+            if(*s == '>') {
+                s++;
+                ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_NOT_EQUAL;
+            }
+            else if(*s == '=') {
+                s++;
+                ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_GREATER_EQUAL;
+            }
+            else
+                ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_GREATER;
+        }
+        else if(*s == '>') {
+            if(*s == '=') {
+                s++;
+                ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_LESS_EQUAL;
+            }
+            else
+                ac->time_group_condition = ALERT_LOOKUP_TIME_GROUP_CONDITION_LESS;
+        }
+
+        while(*s && isspace(*s)) s++;
+
+        if(*s) {
+            if(isdigit(*s) || *s == '.') {
+                ac->time_group_value = str2ndd(s, &s);
+                while(s && *s && isspace(*s)) s++;
+
+                if(!s || *s != ')') {
+                    netdata_log_error("Health configuration at line %zu of file '%s': missing closing parenthesis after number in aggregation method on '%s'",
+                                      line, filename, key);
+                    return 0;
+                }
+            }
+        }
+        else if(*s != ')') {
+            netdata_log_error("Health configuration at line %zu of file '%s': missing closing parenthesis after method on '%s'",
+                              line, filename, key);
+            return 0;
+        }
+
+        s++;
+    }
+
+    switch (ac->time_group) {
+        default:
+            break;
+
+        case RRDR_GROUPING_COUNTIF:
+            if(isnan(ac->time_group_value))
+                ac->time_group_value = 0;
+            break;
+
+        case RRDR_GROUPING_TRIMMED_MEAN:
+        case RRDR_GROUPING_TRIMMED_MEDIAN:
+            if(isnan(ac->time_group_value))
+                ac->time_group_value = 5;
+            break;
+
+        case RRDR_GROUPING_PERCENTILE:
+            if(isnan(ac->time_group_value))
+                ac->time_group_value = 95;
+            break;
+    }
+
     // then is the 'after' time
     key = s;
     while(*s && !isspace(*s)) s++;
     while(*s && isspace(*s)) *s++ = '\0';
 
-    if(!config_parse_duration(key, after)) {
+    if(!config_parse_duration(key, &ac->after)) {
         netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' after group method",
                           line, filename, key);
         return 0;
     }
 
     // sane defaults
-    *every = ABS(*after);
+    ac->update_every = ABS(ac->after);
 
     // now we may have optional parameters
     while(*s) {
@@ -220,7 +294,7 @@ static inline int health_parse_db_lookup(
             while(*s && !isspace(*s)) s++;
             while(*s && isspace(*s)) *s++ = '\0';
 
-            if (!config_parse_duration(value, before)) {
+            if (!config_parse_duration(value, &ac->before)) {
                 netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",
                                   line, filename, value, key);
             }
@@ -230,34 +304,46 @@ static inline int health_parse_db_lookup(
             while(*s && !isspace(*s)) s++;
             while(*s && isspace(*s)) *s++ = '\0';
 
-            if (!config_parse_duration(value, every)) {
+            if (!config_parse_duration(value, &ac->update_every)) {
                 netdata_log_error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",
                                   line, filename, value, key);
             }
         }
         else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
-            *options |= RRDR_OPTION_ABSOLUTE;
+            ac->options |= RRDR_OPTION_ABSOLUTE;
         }
         else if(!strcasecmp(key, "min2max")) {
-            *options |= RRDR_OPTION_MIN2MAX;
+            ac->options |= RRDR_OPTION_DIMS_MIN2MAX;
+        }
+        else if(!strcasecmp(key, "average")) {
+            ac->options |= RRDR_OPTION_DIMS_AVERAGE;
+        }
+        else if(!strcasecmp(key, "min")) {
+            ac->options |= RRDR_OPTION_DIMS_MIN;
+        }
+        else if(!strcasecmp(key, "max")) {
+            ac->options |= RRDR_OPTION_DIMS_MAX;
+        }
+        else if(!strcasecmp(key, "sum")) {
+            ;
         }
         else if(!strcasecmp(key, "null2zero")) {
-            *options |= RRDR_OPTION_NULL2ZERO;
+            ac->options |= RRDR_OPTION_NULL2ZERO;
         }
         else if(!strcasecmp(key, "percentage")) {
-            *options |= RRDR_OPTION_PERCENTAGE;
+            ac->options |= RRDR_OPTION_PERCENTAGE;
         }
         else if(!strcasecmp(key, "unaligned")) {
-            *options |= RRDR_OPTION_NOT_ALIGNED;
+            ac->options |= RRDR_OPTION_NOT_ALIGNED;
         }
         else if(!strcasecmp(key, "anomaly-bit")) {
-            *options |= RRDR_OPTION_ANOMALY_BIT;
+            ac->options |= RRDR_OPTION_ANOMALY_BIT;
         }
         else if(!strcasecmp(key, "match-ids") || !strcasecmp(key, "match_ids")) {
-            *options |= RRDR_OPTION_MATCH_IDS;
+            ac->options |= RRDR_OPTION_MATCH_IDS;
         }
         else if(!strcasecmp(key, "match-names") || !strcasecmp(key, "match_names")) {
-            *options |= RRDR_OPTION_MATCH_NAMES;
+            ac->options |= RRDR_OPTION_MATCH_NAMES;
         }
         else if(!strcasecmp(key, "of")) {
             char *find = NULL;
@@ -266,7 +352,7 @@ static inline int health_parse_db_lookup(
                 if(find) {
                     *find = '\0';
                 }
-                *dimensions = string_strdupz(s);
+                ac->dimensions = string_strdupz(s);
             }
 
             if(!find) {
@@ -340,6 +426,46 @@ static inline void strip_quotes(char *s) {
     }
 }
 
+static void replace_green_red(RRD_ALERT_PROTOTYPE *ap, NETDATA_DOUBLE green, NETDATA_DOUBLE red) {
+    if(!isnan(green)) {
+        STRING *green_str = string_strdupz("green");
+        expression_hardcode_variable(ap->config.calculation, green_str, green);
+        expression_hardcode_variable(ap->config.warning, green_str, green);
+        expression_hardcode_variable(ap->config.critical, green_str, green);
+        string_freez(green_str);
+    }
+
+    if(!isnan(red)) {
+        STRING *red_str = string_strdupz("red");
+        expression_hardcode_variable(ap->config.calculation, red_str, red);
+        expression_hardcode_variable(ap->config.warning, red_str, red);
+        expression_hardcode_variable(ap->config.critical, red_str, red);
+        string_freez(red_str);
+    }
+}
+
+static void dims_grouping_from_rrdr_options(RRD_ALERT_PROTOTYPE *ap) {
+    if(ap->config.options & RRDR_OPTION_DIMS_MIN)
+        ap->config.dims_group = ALERT_LOOKUP_DIMS_MIN;
+    else if(ap->config.options & RRDR_OPTION_DIMS_MAX)
+        ap->config.dims_group = ALERT_LOOKUP_DIMS_MAX;
+    else if(ap->config.options & RRDR_OPTION_DIMS_MIN2MAX)
+        ap->config.dims_group = ALERT_LOOKUP_DIMS_MIN2MAX;
+    else if(ap->config.options & RRDR_OPTION_DIMS_AVERAGE)
+        ap->config.dims_group = ALERT_LOOKUP_DIMS_AVERAGE;
+    else
+        ap->config.dims_group = ALERT_LOOKUP_DIMS_SUM;
+}
+
+static void lookup_data_source_from_rrdr_options(RRD_ALERT_PROTOTYPE *ap) {
+    if(ap->config.options & RRDR_OPTION_PERCENTAGE)
+        ap->config.data_source = ALERT_LOOKUP_DATA_SOURCE_PERCENTAGES;
+    else if(ap->config.options & RRDR_OPTION_ANOMALY_BIT)
+        ap->config.data_source = ALERT_LOOKUP_DATA_SOURCE_ANOMALIES;
+    else
+        ap->config.data_source = ALERT_LOOKUP_DATA_SOURCE_SAMPLES;
+}
+
 #define PARSE_HEALTH_CONFIG_LOG_DUPLICATE_STRING_MSG(ax, member) do {                               \
     if(strcmp(string2str(ax->member), value) != 0)                                                  \
         netdata_log_error(                                                                          \
@@ -391,8 +517,6 @@ static inline void strip_quotes(char *s) {
     }                                                                                               \
 } while(0)
 
-
-
 int health_readfile(const char *filename, void *data __maybe_unused, bool stock_config) {
     netdata_log_debug(D_HEALTH, "Health configuration reading file '%s'", filename);
 
@@ -466,6 +590,8 @@ int health_readfile(const char *filename, void *data __maybe_unused, bool stock_
     RRD_ALERT_PROTOTYPE *ap = NULL;
     struct rrd_alert_config *ac = NULL;
     struct rrd_alert_match *am = NULL;
+    NETDATA_DOUBLE green = NAN;
+    NETDATA_DOUBLE red = NAN;
 
     size_t line = 0, append = 0;
     char *s;
@@ -522,6 +648,9 @@ int health_readfile(const char *filename, void *data __maybe_unused, bool stock_
 
         if((hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) || (hash == hash_template && !strcasecmp(key, HEALTH_