diff options
author | Emmanuel Vasilakis <mrzammler@mm.st> | 2023-05-22 14:14:25 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-22 14:14:25 +0300 |
commit | 10cad04d2d50261dae32b93bd5a5f2dfac5ceb5c (patch) | |
tree | 3b496c27faa377f87de8f71a491bf0d97f0b76c2 /health | |
parent | 128fe427622d43dd2e1397bf37b1536a30779d71 (diff) |
Use chart labels to filter alerts (#14982)
* use chart labels to filter alerts
* add entry to readme
* support chart_label=val val2 val3
* docs updates
* more docs
* use rc not rt
Diffstat (limited to 'health')
-rw-r--r-- | health/REFERENCE.md | 35 | ||||
-rw-r--r-- | health/health_config.c | 92 |
2 files changed, 125 insertions, 2 deletions
diff --git a/health/REFERENCE.md b/health/REFERENCE.md index b95dc852e9..a36edd8cf4 100644 --- a/health/REFERENCE.md +++ b/health/REFERENCE.md @@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation | [`delay`](#alarm-line-delay) | no | Optional hysteresis settings to prevent floods of notifications. | | [`repeat`](#alarm-line-repeat) | no | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode. | | [`options`](#alarm-line-options) | no | Add an option to not clear alarms. | -| [`host labels`](#alarm-line-host-labels) | no | List of labels present on a host. | +| [`host labels`](#alarm-line-host-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. | +| [`chart labels`](#alarm-line-chart-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. | | [`info`](#alarm-line-info) | no | A brief description of the alarm. | The `alarm` or `template` line must be the first line of any entity. @@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t families: sda sdb ``` +Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases. +Please use [`chart labels`](#alarm-line-chart-labels) instead. + #### Alarm line `lookup` This line makes a database lookup to find a value. This result of this lookup is available as `$this`. @@ -696,6 +700,35 @@ host labels: installed = 201* See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples. +#### Alarm line `chart labels` + +Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on +whether these chart labels match or not. + +The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all + +For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having +a value there of which mount point it monitors. + +If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for +it (but you do for all other mount points), you can add the following to the alert's configuration: + +```yaml +chart labels: mount_point=!/mnt/disk1 *` +``` + +The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels, +then the result is an OR between them. i.e. the following: + +```yaml +chart labels: mount_point=/mnt/disk1 device=sda` +``` + +Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name +is specified that does not exist in the chart, the chart won't be matched. + +See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples. + #### Alarm line `info` The info field can contain a small piece of text describing the alarm or template. This will be rendered in diff --git a/health/health_config.c b/health/health_config.c index 38857fc9a5..e21e7785be 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -32,6 +32,7 @@ #define HEALTH_REPEAT_KEY "repeat" #define HEALTH_HOST_LABEL_KEY "host labels" #define HEALTH_FOREACH_KEY "foreach" +#define HEALTH_CHART_LABEL_KEY "chart labels" static inline int health_parse_delay( size_t line, const char *filename, char *string, @@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) { return 1; } +// If needed, add a prefix key to all possible values in the range +static inline char *health_config_add_key_to_values(char *value) { + BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL); + char key[HEALTH_CONF_MAX_LINE + 1]; + char data[HEALTH_CONF_MAX_LINE + 1]; + + char *s = value; + size_t i = 0; + + while(*s) { + if (*s == '=') { + //hold the key + data[i]='\0'; + strncpyz(key, data, HEALTH_CONF_MAX_LINE); + i=0; + } else if (*s == ' ') { + data[i]='\0'; + if (data[0]=='!') + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1); + else + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data); + i=0; + } else { + data[i++] = *s; + } + s++; + } + + data[i]='\0'; + if (data[0]) { + if (data[0]=='!') + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1); + else + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data); + } + + char *final = mallocz(HEALTH_CONF_MAX_LINE + 1); + strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE); + buffer_free(wb); + + return final; +} + static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) { const char *s = value; char buffer[RRDVAR_MAX_LENGTH]; @@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg) string_freez(cfg->host_labels); string_freez(cfg->p_db_lookup_dimensions); string_freez(cfg->p_db_lookup_method); + string_freez(cfg->chart_labels); freez(cfg); } @@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) { hash_delay = 0, hash_options = 0, hash_repeat = 0, - hash_host_label = 0; + hash_host_label = 0, + hash_chart_label = 0; char buffer[HEALTH_CONF_MAX_LINE + 1]; @@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) { hash_options = simple_uhash(HEALTH_OPTIONS_KEY); hash_repeat = simple_uhash(HEALTH_REPEAT_KEY); hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY); + hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY); } FILE *fp = fopen(filename, "r"); @@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) { rc->module_match = string_strdupz(value); rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true); } + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) { + alert_cfg->chart_labels = string_strdupz(value); + if(rc->chart_labels) { + if(strcmp(rrdcalc_chart_labels(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.", + line, filename, rrdcalc_name(rc), key, value, value); + + string_freez(rc->chart_labels); + simple_pattern_free(rc->chart_labels_pattern); + } + + { + char *tmp = simple_pattern_trim_around_equal(value); + char *tmp_2 = health_config_add_key_to_values(tmp); + rc->chart_labels = string_strdupz(tmp_2); + freez(tmp); + freez(tmp_2); + } + rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT, + true); + } else { error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.", line, filename, rrdcalc_name(rc), key); @@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) { rt->host_labels = string_strdupz(tmp); freez(tmp); } + rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL, SIMPLE_PATTERN_EXACT, true); } + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) { + alert_cfg->chart_labels = string_strdupz(value); + if(rt->chart_labels) { + if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value); + + string_freez(rt->chart_labels); + simple_pattern_free(rt->chart_labels_pattern); + } + + { + char *tmp = simple_pattern_trim_around_equal(value); + char *tmp_2 = health_config_add_key_to_values(tmp); + rt->chart_labels = string_strdupz(tmp_2); + freez(tmp); + freez(tmp_2); + } + rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL, + SIMPLE_PATTERN_EXACT, true); + } else { error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.", line, filename, rrdcalctemplate_name(rt), key); |