Use chart labels to filter alerts (#14982)

* use chart labels to filter alerts

* add entry to readme

* support chart_label=val val2 val3

* docs updates

* more docs

* use rc not rt

2 files changed, 125 insertions, 2 deletions

diff --git a/health/REFERENCE.md b/health/REFERENCE.md
index b95dc852e9..a36edd8cf4 100644
--- a/health/REFERENCE.md
+++ b/health/REFERENCE.md
@@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation
 | [`delay`](#alarm-line-delay)                        | no              | Optional hysteresis settings to prevent floods of notifications.                      |
 | [`repeat`](#alarm-line-repeat)                      | no              | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode.  |
 | [`options`](#alarm-line-options)                    | no              | Add an option to not clear alarms.                                                    |
-| [`host labels`](#alarm-line-host-labels)            | no              | List of labels present on a host.                                                     |
+| [`host labels`](#alarm-line-host-labels)            | no              | Restrict an alarm or template to a list of matching labels present on a host.         |
+| [`chart labels`](#alarm-line-chart-labels)          | no              | Restrict an alarm or template to a list of matching labels present on a host.         |
 | [`info`](#alarm-line-info)                          | no              | A brief description of the alarm.                                                           |
 
 The `alarm` or `template` line must be the first line of any entity.
@@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t
 families: sda sdb
 ```
 
+Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases. 
+Please use [`chart labels`](#alarm-line-chart-labels) instead.
+
 #### Alarm line `lookup`
 
 This line makes a database lookup to find a value. This result of this lookup is available as `$this`.
@@ -696,6 +700,35 @@ host labels: installed = 201*
 
 See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
 
+#### Alarm line `chart labels`
+
+Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on
+whether these chart labels match or not.
+
+The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all
+
+For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having
+a value there of which mount point it monitors.
+
+If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for
+it (but you do for all other mount points), you can add the following to the alert's configuration:
+
+```yaml
+chart labels: mount_point=!/mnt/disk1 *`
+```
+
+The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels,
+then the result is an OR between them. i.e. the following:
+
+```yaml
+chart labels: mount_point=/mnt/disk1 device=sda`
+```
+
+Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name
+is specified that does not exist in the chart, the chart won't be matched.
+
+See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
+
 #### Alarm line `info`
 
 The info field can contain a small piece of text describing the alarm or template. This will be rendered in
diff --git a/health/health_config.c b/health/health_config.c
index 38857fc9a5..e21e7785be 100644
--- a/health/health_config.c
+++ b/health/health_config.c
@@ -32,6 +32,7 @@
 #define HEALTH_REPEAT_KEY "repeat"
 #define HEALTH_HOST_LABEL_KEY "host labels"
 #define HEALTH_FOREACH_KEY "foreach"
+#define HEALTH_CHART_LABEL_KEY "chart labels"
 
 static inline int health_parse_delay(
         size_t line, const char *filename, char *string,
@@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) {
     return 1;
 }
 
+// If needed, add a prefix key to all possible values in the range
+static inline char *health_config_add_key_to_values(char *value) {
+    BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL);
+    char key[HEALTH_CONF_MAX_LINE + 1];
+    char data[HEALTH_CONF_MAX_LINE + 1];
+
+    char *s = value;
+    size_t i = 0;
+
+    while(*s) {
+        if (*s == '=') {
+            //hold the key
+            data[i]='\0';
+            strncpyz(key, data, HEALTH_CONF_MAX_LINE);
+            i=0;
+        } else if (*s == ' ') {
+            data[i]='\0';
+            if (data[0]=='!')
+                buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
+            else
+                buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
+            i=0;
+        } else {
+            data[i++] = *s;
+        }
+        s++;
+    }
+
+    data[i]='\0';
+    if (data[0]) {
+        if (data[0]=='!')
+            buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
+        else
+            buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
+    }
+
+    char *final = mallocz(HEALTH_CONF_MAX_LINE + 1);
+    strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE);
+    buffer_free(wb);
+
+    return final;
+}
+
 static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) {
     const char *s = value;
     char buffer[RRDVAR_MAX_LENGTH];
@@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg)
     string_freez(cfg->host_labels);
     string_freez(cfg->p_db_lookup_dimensions);
     string_freez(cfg->p_db_lookup_method);
+    string_freez(cfg->chart_labels);
     freez(cfg);
 }
 
@@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) {
             hash_delay = 0,
             hash_options = 0,
             hash_repeat = 0,
-            hash_host_label = 0;
+            hash_host_label = 0,
+            hash_chart_label = 0;
 
     char buffer[HEALTH_CONF_MAX_LINE + 1];
 
@@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) {
         hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
         hash_repeat = simple_uhash(HEALTH_REPEAT_KEY);
         hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY);
+        hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY);
     }
 
     FILE *fp = fopen(filename, "r");
@@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) {
                 rc->module_match = string_strdupz(value);
                 rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);
             }
+            else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
+                alert_cfg->chart_labels = string_strdupz(value);
+                if(rc->chart_labels) {
+                    if(strcmp(rrdcalc_chart_labels(rc), value) != 0)
+                        error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
+                              line, filename, rrdcalc_name(rc), key, value, value);
+
+                    string_freez(rc->chart_labels);
+                    simple_pattern_free(rc->chart_labels_pattern);
+                }
+
+                {
+                    char *tmp = simple_pattern_trim_around_equal(value);
+                    char *tmp_2 = health_config_add_key_to_values(tmp);
+                    rc->chart_labels = string_strdupz(tmp_2);
+                    freez(tmp);
+                    freez(tmp_2);
+                }
+                rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT,
+                                                                true);
+            }
             else {
                 error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",
                         line, filename, rrdcalc_name(rc), key);
@@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) {
                     rt->host_labels = string_strdupz(tmp);
                     freez(tmp);
                 }
+
                 rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL,
                                                                 SIMPLE_PATTERN_EXACT, true);
             }
+            else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
+                alert_cfg->chart_labels = string_strdupz(value);
+                if(rt->chart_labels) {
+                    if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0)
+                        error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+                              line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value);
+
+                    string_freez(rt->chart_labels);
+                    simple_pattern_free(rt->chart_labels_pattern);
+                }
+
+                {
+                    char *tmp = simple_pattern_trim_around_equal(value);
+                    char *tmp_2 = health_config_add_key_to_values(tmp);
+                    rt->chart_labels = string_strdupz(tmp_2);
+                    freez(tmp);
+                    freez(tmp_2);
+                }
+                rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL,
+                                                                SIMPLE_PATTERN_EXACT, true);
+            }
             else {
                 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
                         line, filename, rrdcalctemplate_name(rt), key);