From 10cad04d2d50261dae32b93bd5a5f2dfac5ceb5c Mon Sep 17 00:00:00 2001 From: Emmanuel Vasilakis Date: Mon, 22 May 2023 14:14:25 +0300 Subject: Use chart labels to filter alerts (#14982) * use chart labels to filter alerts * add entry to readme * support chart_label=val val2 val3 * docs updates * more docs * use rc not rt --- database/rrdcalc.c | 6 +++ database/rrdcalc.h | 5 +++ database/rrdcalctemplate.c | 7 ++++ database/rrdcalctemplate.h | 4 ++ database/sqlite/sqlite_health.c | 1 + health/REFERENCE.md | 35 +++++++++++++++- health/health_config.c | 92 ++++++++++++++++++++++++++++++++++++++++- 7 files changed, 148 insertions(+), 2 deletions(-) diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 3ee8719c07..948ebe8a55 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -369,6 +369,10 @@ static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) { st->rrdhost->rrdlabels, rc->host_labels_pattern, '=', NULL)) return false; + if (st->rrdlabels && rc->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed( + st->rrdlabels, rc->chart_labels_pattern, '=', NULL)) + return false; + return true; } @@ -605,11 +609,13 @@ static void rrdcalc_free_internals(RRDCALC *rc) { string_freez(rc->host_labels); string_freez(rc->module_match); string_freez(rc->plugin_match); + string_freez(rc->chart_labels); simple_pattern_free(rc->foreach_dimension_pattern); simple_pattern_free(rc->host_labels_pattern); simple_pattern_free(rc->module_pattern); simple_pattern_free(rc->plugin_pattern); + simple_pattern_free(rc->chart_labels_pattern); } static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) { diff --git a/database/rrdcalc.h b/database/rrdcalc.h index c6d6fd4e64..3b48d74ec2 100644 --- a/database/rrdcalc.h +++ b/database/rrdcalc.h @@ -109,6 +109,9 @@ struct rrdcalc { STRING *host_labels; // the label read from an alarm file SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels + STRING *chart_labels; // the chart label read from an alarm file + SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels + // ------------------------------------------------------------------------ // runtime information @@ -168,6 +171,7 @@ struct rrdcalc { #define rrdcalc_dimensions(rc) string2str((rc)->dimensions) #define rrdcalc_foreachdim(rc) string2str((rc)->foreach_dimension) #define rrdcalc_host_labels(rc) string2str((rc)->host_labels) +#define rrdcalc_chart_labels(rc) string2str((rc)->chart_labels) #define foreach_rrdcalc_in_rrdhost_read(host, rc) \ dfe_start_read((host)->rrdcalc_root_index, rc) \ @@ -206,6 +210,7 @@ struct alert_config { STRING *options; STRING *repeat; STRING *host_labels; + STRING *chart_labels; STRING *p_db_lookup_dimensions; STRING *p_db_lookup_method; diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c index 4dacb6c7b0..53630f99c6 100644 --- a/database/rrdcalctemplate.c +++ b/database/rrdcalctemplate.c @@ -51,6 +51,11 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR '=', NULL)) return false; + if(st->rrdlabels && rt->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdlabels, + rt->chart_labels_pattern, + '=', NULL)) + return false; + return true; } @@ -120,8 +125,10 @@ static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) { string_freez(rt->dimensions); string_freez(rt->foreach_dimension); string_freez(rt->host_labels); + string_freez(rt->chart_labels); simple_pattern_free(rt->foreach_dimension_pattern); simple_pattern_free(rt->host_labels_pattern); + simple_pattern_free(rt->chart_labels_pattern); } void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt) { diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h index 22cfe06e8f..965a818a1b 100644 --- a/database/rrdcalctemplate.h +++ b/database/rrdcalctemplate.h @@ -74,6 +74,9 @@ struct rrdcalctemplate { STRING *host_labels; // the label read from an alarm file SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels + STRING *chart_labels; // the chart label read from an alarm file + SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels + // ------------------------------------------------------------------------ // expressions related to the alarm @@ -107,6 +110,7 @@ struct rrdcalctemplate { #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions) #define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreach_dimension) #define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels) +#define rrdcalctemplate_chart_labels(rt) string2str((rt)->chart_labels) #define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after) diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c index dd08f63ecc..36a29d2e47 100644 --- a/database/sqlite/sqlite_health.c +++ b/database/sqlite/sqlite_health.c @@ -1070,6 +1070,7 @@ int alert_hash_and_store_config( DIGEST_ALERT_CONFIG_VAL(cfg->options); DIGEST_ALERT_CONFIG_VAL(cfg->repeat); DIGEST_ALERT_CONFIG_VAL(cfg->host_labels); + DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels); EVP_DigestFinal_ex(evpctx, hash_value, &hash_len); EVP_MD_CTX_destroy(evpctx); diff --git a/health/REFERENCE.md b/health/REFERENCE.md index b95dc852e9..a36edd8cf4 100644 --- a/health/REFERENCE.md +++ b/health/REFERENCE.md @@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation | [`delay`](#alarm-line-delay) | no | Optional hysteresis settings to prevent floods of notifications. | | [`repeat`](#alarm-line-repeat) | no | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode. | | [`options`](#alarm-line-options) | no | Add an option to not clear alarms. | -| [`host labels`](#alarm-line-host-labels) | no | List of labels present on a host. | +| [`host labels`](#alarm-line-host-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. | +| [`chart labels`](#alarm-line-chart-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. | | [`info`](#alarm-line-info) | no | A brief description of the alarm. | The `alarm` or `template` line must be the first line of any entity. @@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t families: sda sdb ``` +Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases. +Please use [`chart labels`](#alarm-line-chart-labels) instead. + #### Alarm line `lookup` This line makes a database lookup to find a value. This result of this lookup is available as `$this`. @@ -696,6 +700,35 @@ host labels: installed = 201* See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples. +#### Alarm line `chart labels` + +Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on +whether these chart labels match or not. + +The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all + +For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having +a value there of which mount point it monitors. + +If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for +it (but you do for all other mount points), you can add the following to the alert's configuration: + +```yaml +chart labels: mount_point=!/mnt/disk1 *` +``` + +The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels, +then the result is an OR between them. i.e. the following: + +```yaml +chart labels: mount_point=/mnt/disk1 device=sda` +``` + +Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name +is specified that does not exist in the chart, the chart won't be matched. + +See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples. + #### Alarm line `info` The info field can contain a small piece of text describing the alarm or template. This will be rendered in diff --git a/health/health_config.c b/health/health_config.c index 38857fc9a5..e21e7785be 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -32,6 +32,7 @@ #define HEALTH_REPEAT_KEY "repeat" #define HEALTH_HOST_LABEL_KEY "host labels" #define HEALTH_FOREACH_KEY "foreach" +#define HEALTH_CHART_LABEL_KEY "chart labels" static inline int health_parse_delay( size_t line, const char *filename, char *string, @@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) { return 1; } +// If needed, add a prefix key to all possible values in the range +static inline char *health_config_add_key_to_values(char *value) { + BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL); + char key[HEALTH_CONF_MAX_LINE + 1]; + char data[HEALTH_CONF_MAX_LINE + 1]; + + char *s = value; + size_t i = 0; + + while(*s) { + if (*s == '=') { + //hold the key + data[i]='\0'; + strncpyz(key, data, HEALTH_CONF_MAX_LINE); + i=0; + } else if (*s == ' ') { + data[i]='\0'; + if (data[0]=='!') + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1); + else + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data); + i=0; + } else { + data[i++] = *s; + } + s++; + } + + data[i]='\0'; + if (data[0]) { + if (data[0]=='!') + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1); + else + buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data); + } + + char *final = mallocz(HEALTH_CONF_MAX_LINE + 1); + strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE); + buffer_free(wb); + + return final; +} + static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) { const char *s = value; char buffer[RRDVAR_MAX_LENGTH]; @@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg) string_freez(cfg->host_labels); string_freez(cfg->p_db_lookup_dimensions); string_freez(cfg->p_db_lookup_method); + string_freez(cfg->chart_labels); freez(cfg); } @@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) { hash_delay = 0, hash_options = 0, hash_repeat = 0, - hash_host_label = 0; + hash_host_label = 0, + hash_chart_label = 0; char buffer[HEALTH_CONF_MAX_LINE + 1]; @@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) { hash_options = simple_uhash(HEALTH_OPTIONS_KEY); hash_repeat = simple_uhash(HEALTH_REPEAT_KEY); hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY); + hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY); } FILE *fp = fopen(filename, "r"); @@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) { rc->module_match = string_strdupz(value); rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true); } + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) { + alert_cfg->chart_labels = string_strdupz(value); + if(rc->chart_labels) { + if(strcmp(rrdcalc_chart_labels(rc), value) != 0) + error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.", + line, filename, rrdcalc_name(rc), key, value, value); + + string_freez(rc->chart_labels); + simple_pattern_free(rc->chart_labels_pattern); + } + + { + char *tmp = simple_pattern_trim_around_equal(value); + char *tmp_2 = health_config_add_key_to_values(tmp); + rc->chart_labels = string_strdupz(tmp_2); + freez(tmp); + freez(tmp_2); + } + rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT, + true); + } else { error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.", line, filename, rrdcalc_name(rc), key); @@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) { rt->host_labels = string_strdupz(tmp); freez(tmp); } + rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL, SIMPLE_PATTERN_EXACT, true); } + else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) { + alert_cfg->chart_labels = string_strdupz(value); + if(rt->chart_labels) { + if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0) + error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value); + + string_freez(rt->chart_labels); + simple_pattern_free(rt->chart_labels_pattern); + } + + { + char *tmp = simple_pattern_trim_around_equal(value); + char *tmp_2 = health_config_add_key_to_values(tmp); + rt->chart_labels = string_strdupz(tmp_2); + freez(tmp); + freez(tmp_2); + } + rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL, + SIMPLE_PATTERN_EXACT, true); + } else { error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.", line, filename, rrdcalctemplate_name(rt), key); -- cgit v1.2.3