Use chart labels to filter alerts (#14982)

* use chart labels to filter alerts

* add entry to readme

* support chart_label=val val2 val3

* docs updates

* more docs

* use rc not rt

7 files changed, 148 insertions, 2 deletions

diff --git a/database/rrdcalc.c b/database/rrdcalc.c
index 3ee8719c07..948ebe8a55 100644
--- a/database/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -369,6 +369,10 @@ static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) {
             st->rrdhost->rrdlabels, rc->host_labels_pattern, '=', NULL))
         return false;
 
+    if (st->rrdlabels && rc->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(
+            st->rrdlabels, rc->chart_labels_pattern, '=', NULL))
+        return false;
+
     return true;
 }
 
@@ -605,11 +609,13 @@ static void rrdcalc_free_internals(RRDCALC *rc) {
     string_freez(rc->host_labels);
     string_freez(rc->module_match);
     string_freez(rc->plugin_match);
+    string_freez(rc->chart_labels);
 
     simple_pattern_free(rc->foreach_dimension_pattern);
     simple_pattern_free(rc->host_labels_pattern);
     simple_pattern_free(rc->module_pattern);
     simple_pattern_free(rc->plugin_pattern);
+    simple_pattern_free(rc->chart_labels_pattern);
 }
 
 static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) {
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
index c6d6fd4e64..3b48d74ec2 100644
--- a/database/rrdcalc.h
+++ b/database/rrdcalc.h
@@ -109,6 +109,9 @@ struct rrdcalc {
     STRING *host_labels;                 // the label read from an alarm file
     SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
 
+    STRING *chart_labels;                 // the chart label read from an alarm file
+    SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels
+
     // ------------------------------------------------------------------------
     // runtime information
 
@@ -168,6 +171,7 @@ struct rrdcalc {
 #define rrdcalc_dimensions(rc) string2str((rc)->dimensions)
 #define rrdcalc_foreachdim(rc) string2str((rc)->foreach_dimension)
 #define rrdcalc_host_labels(rc) string2str((rc)->host_labels)
+#define rrdcalc_chart_labels(rc) string2str((rc)->chart_labels)
 
 #define foreach_rrdcalc_in_rrdhost_read(host, rc) \
     dfe_start_read((host)->rrdcalc_root_index, rc) \
@@ -206,6 +210,7 @@ struct alert_config {
     STRING *options;
     STRING *repeat;
     STRING *host_labels;
+    STRING *chart_labels;
 
     STRING *p_db_lookup_dimensions;
     STRING *p_db_lookup_method;
diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c
index 4dacb6c7b0..53630f99c6 100644
--- a/database/rrdcalctemplate.c
+++ b/database/rrdcalctemplate.c
@@ -51,6 +51,11 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR
                                                                                             '=', NULL))
         return false;
 
+    if(st->rrdlabels && rt->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdlabels,
+                                                                                            rt->chart_labels_pattern,
+                                                                                            '=', NULL))
+        return false;
+
     return true;
 }
 
@@ -120,8 +125,10 @@ static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) {
     string_freez(rt->dimensions);
     string_freez(rt->foreach_dimension);
     string_freez(rt->host_labels);
+    string_freez(rt->chart_labels);
     simple_pattern_free(rt->foreach_dimension_pattern);
     simple_pattern_free(rt->host_labels_pattern);
+    simple_pattern_free(rt->chart_labels_pattern);
 }
 
 void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt) {
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 22cfe06e8f..965a818a1b 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -74,6 +74,9 @@ struct rrdcalctemplate {
     STRING *host_labels;                 // the label read from an alarm file
     SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
 
+    STRING *chart_labels;                 // the chart label read from an alarm file
+    SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels
+
     // ------------------------------------------------------------------------
     // expressions related to the alarm
 
@@ -107,6 +110,7 @@ struct rrdcalctemplate {
 #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)
 #define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreach_dimension)
 #define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels)
+#define rrdcalctemplate_chart_labels(rt) string2str((rt)->chart_labels)
 
 #define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after)
 
diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c
index dd08f63ecc..36a29d2e47 100644
--- a/database/sqlite/sqlite_health.c
+++ b/database/sqlite/sqlite_health.c
@@ -1070,6 +1070,7 @@ int alert_hash_and_store_config(
     DIGEST_ALERT_CONFIG_VAL(cfg->options);
     DIGEST_ALERT_CONFIG_VAL(cfg->repeat);
     DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);
+    DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);
 
     EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
     EVP_MD_CTX_destroy(evpctx);
diff --git a/health/REFERENCE.md b/health/REFERENCE.md
index b95dc852e9..a36edd8cf4 100644
--- a/health/REFERENCE.md
+++ b/health/REFERENCE.md
@@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation
 | [`delay`](#alarm-line-delay)                        | no              | Optional hysteresis settings to prevent floods of notifications.                      |
 | [`repeat`](#alarm-line-repeat)                      | no              | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode.  |
 | [`options`](#alarm-line-options)                    | no              | Add an option to not clear alarms.                                                    |
-| [`host labels`](#alarm-line-host-labels)            | no              | List of labels present on a host.                                                     |
+| [`host labels`](#alarm-line-host-labels)            | no              | Restrict an alarm or template to a list of matching labels present on a host.         |
+| [`chart labels`](#alarm-line-chart-labels)          | no              | Restrict an alarm or template to a list of matching labels present on a host.         |
 | [`info`](#alarm-line-info)                          | no              | A brief description of the alarm.                                                           |
 
 The `alarm` or `template` line must be the first line of any entity.
@@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t
 families: sda sdb
 ```
 
+Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases. 
+Please use [`chart labels`](#alarm-line-chart-labels) instead.
+
 #### Alarm line `lookup`
 
 This line makes a database lookup to find a value. This result of this lookup is available as `$this`.
@@ -696,6 +700,35 @@ host labels: installed = 201*
 
 See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
 
+#### Alarm line `chart labels`
+
+Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on
+whether these chart labels match or not.
+
+The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all
+
+For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having
+a value there of which mount point it monitors.
+
+If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for
+it (but you do for all other mount points), you can add the following to the alert's configuration:
+
+```yaml
+chart labels: mount_point=!/mnt/disk1 *`
+```
+
+The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels,
+then the result is an OR between them. i.e. the following:
+
+```yaml
+chart labels: mount_point=/mnt/disk1 device=sda`
+```
+
+Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name
+is specified that does not exist in the chart, the chart won't be matched.
+
+See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
+
 #### Alarm line `info`
 
 The info field can contain a small piece of text describing the alarm or template. This will be rendered in
diff --git a/health/health_config.c b/health/health_config.c
index 38857fc9a5..e21e7785be 100644
--- a/health/health_config.c
+++ b/health/health_config.c
@@ -32,6 +32,7 @@
 #define HEALTH_REPEAT_KEY "repeat"
 #define HEALTH_HOST_LABEL_KEY "host labels"
 #define HEALTH_FOREACH_KEY "foreach"
+#define HEALTH_CHART_LABEL_KEY "chart labels"
 
 static inline int health_parse_delay(
         size_t line, const char *filename, char *string,
@@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) {
     return 1;
 }
 
+// If needed, add a prefix key to all possible values in the range
+static inline char *health_config_add_key_to_values(char *value) {
+    BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL);
+    char key[HEALTH_CONF_MAX_LINE + 1];
+    char data[HEALTH_CONF_MAX_LINE + 1];
+
+    char *s = value;
+    size_t i = 0;
+
+    while(*s) {
+        if (*s == '=') {
+            //hold the key
+            data[i]='\0';
+            strncpyz(key, data, HEALTH_CONF_MAX_LINE);
+            i=0;
+        } else if (*s == ' ') {
+            data[i]='\0';
+            if (data[0]=='!')
+                buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
+            else
+                buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
+            i=0;
+        } else {
+            data[i++] = *s;
+        }
+        s++;
+    }
+
+    data[i]='\0';
+    if (data[0]) {
+        if (data[0]=='!')
+            buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
+        else
+            buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
+    }
+
+    char *final = mallocz(HEALTH_CONF_MAX_LINE + 1);
+    strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE);
+    buffer_free(wb);
+
+    return final;
+}
+
 static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) {
     const char *s = value;
     char buffer[RRDVAR_MAX_LENGTH];
@@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg)
     string_freez(cfg->host_labels);
     string_freez(cfg->p_db_lookup_dimensions);
     string_freez(cfg->p_db_lookup_method);
+    string_freez(cfg->chart_labels);
     freez(cfg);
 }
 
@@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) {
             hash_delay = 0,
             hash_options = 0,
             hash_repeat = 0,
-            hash_host_label = 0;
+            hash_host_label = 0,
+            hash_chart_label = 0;
 
     char buffer[HEALTH_CONF_MAX_LINE + 1];
 
@@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) {
         hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
         hash_repeat = simple_uhash(HEALTH_REPEAT_KEY);
         hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY);
+        hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY);
     }
 
     FILE *fp = fopen(filename, "r");
@@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) {
                 rc->module_match = string_strdupz(value);
                 rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);
             }
+            else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
+                alert_cfg->chart_labels = string_strdupz(value);
+                if(rc->chart_labels) {
+                    if(strcmp(rrdcalc_chart_labels(rc), value) != 0)
+                        error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
+                              line, filename, rrdcalc_name(rc), key, value, value);
+
+                    string_freez(rc->chart_labels);
+                    simple_pattern_free(rc->chart_labels_pattern);
+                }
+
+                {
+                    char *tmp = simple_pattern_trim_around_equal(value);
+                    char *tmp_2 = health_config_add_key_to_values(tmp);
+                    rc->chart_labels = string_strdupz(tmp_2);
+                    freez(tmp);
+                    freez(tmp_2);
+                }
+                rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT,
+                                                                true);
+            }
             else {
                 error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",
                         line, filename, rrdcalc_name(rc), key);
@@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) {
                     rt->host_labels = string_strdupz(tmp);
                     freez(tmp);
                 }
+
                 rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL,
                                                                 SIMPLE_PATTERN_EXACT, true);
             }
+            else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
+                alert_cfg->chart_labels = string_strdupz(value);
+                if(rt->chart_labels) {
+                    if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0)
+                        error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+                              line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value);
+
+                    string_freez(rt->chart_labels);
+                    simple_pattern_free(rt->chart_labels_pattern);
+                }
+
+                {
+                    char *tmp = simple_pattern_trim_around_equal(value);
+                    char *tmp_2 = health_config_add_key_to_values(tmp);
+                    rt->chart_labels = string_strdupz(tmp_2);
+                    freez(tmp);
+                    freez(tmp_2);
+                }
+                rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL,
+                                                                SIMPLE_PATTERN_EXACT, true);
+            }
             else {
                 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
                         line, filename, rrdcalctemplate_name(rt), key);