diff options
36 files changed, 906 insertions, 76 deletions
diff --git a/.gitignore b/.gitignore index 05255d1784..284fcda506 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,7 @@ system/netdata-freebsd system/edit-config daemon/anonymous-statistics.sh +daemon/get-kubernetes-labels.sh health/notifications/alarm-notify.sh collectors/cgroups.plugin/cgroup-name.sh @@ -1,3 +1,4 @@ + # The build system We are currently migrating from `autotools` to `CMake` as a build-system. This document diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 6383b559ed..d3102f695e 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -13,9 +13,11 @@ dist_noinst_DATA = \ README.md \ config/README.md \ anonymous-statistics.sh.in \ + get-kubernetes-labels.sh.in \ $(NULL) dist_plugins_SCRIPTS = \ anonymous-statistics.sh \ system-info.sh \ + get-kubernetes-labels.sh \ $(NULL) diff --git a/daemon/commands.c b/daemon/commands.c index 6847e9f0be..5eb22a6693 100644 --- a/daemon/commands.c +++ b/daemon/commands.c @@ -40,6 +40,7 @@ static cmd_status_t cmd_save_database_execute(char *args, char **message); static cmd_status_t cmd_reopen_logs_execute(char *args, char **message); static cmd_status_t cmd_exit_execute(char *args, char **message); static cmd_status_t cmd_fatal_execute(char *args, char **message); +static cmd_status_t cmd_reload_labels_execute(char *args, char **message); static command_info_t command_info_array[] = { {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu @@ -48,6 +49,7 @@ static command_info_t command_info_array[] = { {"reopen-logs", cmd_reopen_logs_execute, CMD_TYPE_ORTHOGONAL}, // Close and reopen log files {"shutdown-agent", cmd_exit_execute, CMD_TYPE_EXCLUSIVE}, // exit cleanly {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY}, // exit with fatal error + {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL}, // reload the labels }; /* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */ @@ -97,6 +99,8 @@ static cmd_status_t cmd_help_execute(char *args, char **message) " Show this help menu.\n" "reload-health\n" " Reload health configuration.\n" + "reload-labels\n" + " Reload all labels.\n" "save-database\n" " Save internal DB to disk for memory mode save.\n" "reopen-logs\n" @@ -172,6 +176,28 @@ static cmd_status_t cmd_fatal_execute(char *args, char **message) return CMD_STATUS_SUCCESS; } +static cmd_status_t cmd_reload_labels_execute(char *args, char **message) +{ + (void)args; + info("COMMAND: reloading host labels."); + reload_host_labels(); + + BUFFER *wb = buffer_create(10); + + netdata_rwlock_rdlock(&localhost->labels_rwlock); + struct label *l=localhost->labels; + while (l != NULL) { + buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value); + l = l->next; + } + netdata_rwlock_unlock(&localhost->labels_rwlock); + + (*message)=strdupz(buffer_tostring(wb)); + buffer_free(wb); + + return CMD_STATUS_SUCCESS; +} + static void cmd_lock_exclusive(unsigned index) { (void)index; diff --git a/daemon/commands.h b/daemon/commands.h index 55cd6455d6..6de4084217 100644 --- a/daemon/commands.h +++ b/daemon/commands.h @@ -19,6 +19,7 @@ typedef enum cmd { CMD_REOPEN_LOGS, CMD_EXIT, CMD_FATAL, + CMD_RELOAD_LABELS, CMD_TOTAL_COMMANDS } cmd_t; diff --git a/daemon/common.h b/daemon/common.h index c04c8ad04f..5202a53082 100644 --- a/daemon/common.h +++ b/daemon/common.h @@ -8,7 +8,7 @@ // ---------------------------------------------------------------------------- // shortcuts for the default netdata configuration -#define config_load(filename, overwrite_used) appconfig_load(&netdata_config, filename, overwrite_used) +#define config_load(filename, overwrite_used, section) appconfig_load(&netdata_config, filename, overwrite_used, section) #define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value) #define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value) #define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value) diff --git a/daemon/get-kubernetes-labels.sh.in b/daemon/get-kubernetes-labels.sh.in new file mode 100644 index 0000000000..805d027b86 --- /dev/null +++ b/daemon/get-kubernetes-labels.sh.in @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Checks if netdata is running in a kubernetes pod and fetches that pod's labels + +if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ] && [ -n "${MY_POD_NAMESPACE}" ] && [ -n "${MY_POD_NAME}" ]; then + if command -v jq >/dev/null 2>&1; then + KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)" + URL="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME" + curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" "$URL" | + jq -r '.metadata.labels' | grep ':' | tr -d '," ' + exit 0 + else + echo "jq command not available. Please install jq to get host labels for kubernetes pods." + exit 1 + fi +else + exit 0 +fi diff --git a/daemon/main.c b/daemon/main.c index f8c6d84d45..ece8feb447 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -703,20 +703,20 @@ static int load_netdata_conf(char *filename, char overwrite_used) { int ret = 0; if(filename && *filename) { - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) error("CONFIG: cannot load config file '%s'.", filename); } else { filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf"); - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) { info("CONFIG: cannot load user config '%s'. Will try the stock version.", filename); freez(filename); filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf"); - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename); } @@ -1271,6 +1271,9 @@ int main(int argc, char **argv) { error_log_limit_reset(); + // Load host labels + reload_host_labels(); + // ------------------------------------------------------------------------ // spawn the threads diff --git a/daemon/system-info.sh b/daemon/system-info.sh index 6fab741fde..7b091a9e54 100755 --- a/daemon/system-info.sh +++ b/daemon/system-info.sh @@ -107,7 +107,7 @@ if [ "${CONTAINER}" = "unknown" ]; then fi fi -echo "NETDATA_SYSTEM_OS_NAME=\"${NAME}\"" +echo "NETDATA_SYSTEM_OS_NAME=${NAME}" echo "NETDATA_SYSTEM_OS_ID=${ID}" echo "NETDATA_SYSTEM_OS_ID_LIKE=${ID_LIKE}" echo "NETDATA_SYSTEM_OS_VERSION=${VERSION}" diff --git a/database/rrd.h b/database/rrd.h index c75b0deebe..432341d75d 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -148,6 +148,25 @@ typedef enum rrddim_flags { #define rrddim_flag_clear(rd, flag) (rd)->flags &= ~(flag) #endif +typedef enum label_source { + LABEL_SOURCE_AUTO = 0, + LABEL_SOURCE_NETDATA_CONF = 1, + LABEL_SOURCE_DOCKER = 2, + LABEL_SOURCE_ENVIRONMENT = 3, + LABEL_SOURCE_KUBERNETES = 4 +} LABEL_SOURCE; + +struct label { + char *key, *value; + uint32_t key_hash; + LABEL_SOURCE label_source; + struct label *next; +}; + +char *translate_label_source(LABEL_SOURCE l); +struct label *create_label(char *key, char *value, LABEL_SOURCE label_source); +struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source); +void reload_host_labels(); // ---------------------------------------------------------------------------- // RRD DIMENSION - this is a metric @@ -727,6 +746,11 @@ struct rrdhost { netdata_rwlock_t rrdhost_rwlock; // lock for this RRDHOST (protects rrdset_root linked list) // ------------------------------------------------------------------------ + // Support for host-level labels + struct label *labels; + netdata_rwlock_t labels_rwlock; // lock for the label list + + // ------------------------------------------------------------------------ // indexes avl_tree_lock rrdset_root_index; // the host's charts index (by id) diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 9f16ce3740..a0a7f51141 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -559,6 +559,8 @@ void rrdcalc_free(RRDCALC *rc) { freez(rc->units); freez(rc->info); simple_pattern_free(rc->spdim); + freez(rc->labels); + simple_pattern_free(rc->splabels); freez(rc); } @@ -603,6 +605,91 @@ void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) { rrdcalc_free(rc); } +void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) { + + if(unlikely(rc == host->alarms_with_foreach)) + host->alarms_with_foreach = rc->next; + else { + RRDCALC *t; + for(t = host->alarms_with_foreach; t && t->next != rc; t = t->next) ; + if(t) { + t->next = rc->next; + rc->next = NULL; + } + else + error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); + } + + rrdcalc_free(rc); +} + +static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) { + RRDCALC *rc = alarms; + while (rc) { + if (!rc->labels) { + rc = rc->next; + continue; + } + + char cmp[CONFIG_FILE_LINE_MAX+1]; + struct label *move = host->labels; + while(move) { + snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value); + if (simple_pattern_matches(rc->splabels, move->key) || + simple_pattern_matches(rc->splabels, cmp)) { + break; + } + + move = move->next; + } + + RRDCALC *next = rc->next; + if(!move) { + info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'", + rc->name, + host->hostname, + rc->labels); + + if(host->alarms == alarms) { + rrdcalc_unlink_and_free(host, rc); + } else + rrdcalc_foreach_unlink_and_free(host, rc); + + } + + rc = next; + } +} + +void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) { + netdata_rwlock_rdlock(&host->labels_rwlock); + + rrdcalc_labels_unlink_alarm_loop(host, host->alarms); + rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach); + + netdata_rwlock_unlock(&host->labels_rwlock); +} + +void rrdcalc_labels_unlink() { + rrd_rdlock(); + + RRDHOST *host; + rrdhost_foreach_read(host) { + if (unlikely(!host->health_enabled)) + continue; + + if (host->labels) { + rrdhost_wrlock(host); + + rrdcalc_labels_unlink_alarm_from_host(host); + + rrdhost_unlock(host); + } + } + + rrd_unlock(); +} + // ---------------------------------------------------------------------------- // Alarm diff --git a/database/rrdcalc.h b/database/rrdcalc.h index e0b6325971..e47bba1cd6 100644 --- a/database/rrdcalc.h +++ b/database/rrdcalc.h @@ -91,6 +91,11 @@ struct rrdcalc { uint32_t crit_repeat_every; // interval between repeating critical notifications // ------------------------------------------------------------------------ + // Labels settings + char *labels; // the label read from an alarm file + SIMPLE_PATTERN *splabels; // the simple pattern of labels + + // ------------------------------------------------------------------------ // runtime information RRDCALC_STATUS old_status; // the old status of the alarm @@ -157,6 +162,9 @@ extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc); extern void dimension_remove_pipe_comma(char *str); extern char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen); +extern void rrdcalc_labels_unlink(); +extern void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host); + static inline int rrdcalc_isrepeating(RRDCALC *rc) { if (unlikely(rc->warn_repeat_every > 0 || rc->crit_repeat_every > 0)) { return 1; diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c index f7a0855611..58ec95a087 100644 --- a/database/rrdcalctemplate.c +++ b/database/rrdcalctemplate.c @@ -4,6 +4,45 @@ #include "rrd.h" // ---------------------------------------------------------------------------- + +static int rrdcalctemplate_is_there_label_restriction(RRDCALCTEMPLATE *rt, RRDHOST *host) { + if(!rt->labels) + return 0; + + errno = 0; + struct label *move = host->labels; + char cmp[CONFIG_FILE_LINE_MAX+1]; + + int ret; + if(move) { + netdata_rwlock_rdlock(&host->labels_rwlock); + while(move) { + snprintfz(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value); + if (simple_pattern_matches(rt->splabels, move->key) || + simple_pattern_matches(rt->splabels, cmp)) { + break; + } + move = move->next; + } + netdata_rwlock_unlock(&host->labels_rwlock); + + if(!move) { + error("Health template '%s' cannot be applied, because the host %s does not have the label(s) '%s'", + rt->name, + host->hostname, + rt->labels + ); + ret = 1; + } else { + ret = 0; + } + } else { + ret =0; + } + + return ret; +} + // RRDCALCTEMPLATE management /** * RRDCALC TEMPLATE LINK MATCHING @@ -14,13 +53,18 @@ void rrdcalctemplate_link_matching_test(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host ) { if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context) && (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) { - RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id); - if(unlikely(!rc)) - info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname); + if (!rrdcalctemplate_is_there_label_restriction(rt, host)) { + RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id); + if (unlikely(!rc)) + info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", + rt->name, st->id, host->hostname); #ifdef NETDATA_INTERNAL_CHECKS - else if(rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late - error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id); + else if (rc->rrdset != st && + !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late + error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", + rc->chart ? rc->chart : "NOCHART", rc->name, st->id); #endif + } } } @@ -56,7 +100,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { freez(rt->info); freez(rt->dimensions); freez(rt->foreachdim); + freez(rt->labels); simple_pattern_free(rt->spdim); + simple_pattern_free(rt->splabels); freez(rt); } diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h index 676b4cf645..9cfd09846e 100644 --- a/database/rrdcalctemplate.h +++ b/database/rrdcalctemplate.h @@ -59,6 +59,11 @@ struct rrdcalctemplate { uint32_t crit_repeat_every; // interval between repeating critical notifications // ------------------------------------------------------------------------ + // Labels settings + char *labels; // the label read from an alarm file + SIMPLE_PATTERN *splabels; // the simple pattern of labels + + // ------------------------------------------------------------------------ // expressions related to the alarm EVAL_EXPRESSION *calculation; diff --git a/database/rrdhost.c b/database/rrdhost.c index 737e066c9d..150596671f 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -709,6 +709,248 @@ void rrdhost_save_charts(RRDHOST *host) { rrdhost_unlock(host); } +static int is_valid_label_key(char *key) { + //Prometheus exporter + if(!strcmp(key, "chart") || !strcmp(key, "family") || !strcmp(key, "dimension")) + return 0; + + //Netdata and Prometheus internal + if (*key == '_') + return 0; + + while(*key) { + if(!(isdigit(*key) || isalpha(*key) || *key == '.' || *key == '_' || *key == '-')) + return 0; + + key++; + } + + return 1; +} + +char *translate_label_source(LABEL_SOURCE l) { + switch (l) { + case LABEL_SOURCE_AUTO: + return "AUTO"; + case LABEL_SOURCE_NETDATA_CONF: + return "NETDATA.CONF"; + case LABEL_SOURCE_DOCKER : + return "DOCKER"; + case LABEL_SOURCE_ENVIRONMENT : + return "ENVIRONMENT"; + case LABEL_SOURCE_KUBERNETES : + return "KUBERNETES"; + default: + return "Invalid label source"; + } +} + +struct label *load_auto_labels() +{ + struct label *label_list = NULL; + + if (localhost->system_info->os_name) + label_list = + add_label_to_list(label_list, "_os_name", localhost->system_info->os_name, LABEL_SOURCE_AUTO); + + if (localhost->system_info->os_version) + label_list = + add_label_to_list(label_list, "_os_version", localhost->system_info->os_version, LABEL_SOURCE_AUTO); + + if (localhost->system_info->kernel_version) + label_list = + add_label_to_list(label_list, "_kernel_version", localhost->system_info->kernel_version, LABEL_SOURCE_AUTO); + + if (localhost->system_info->architecture) + label_list = + add_label_to_list(label_list, "_architecture", localhost->system_info->architecture, LABEL_SOURCE_AUTO); + + if (localhost->system_info->virtualization) + label_list = + add_label_to_list(label_list, "_virtualization", localhost->system_info->virtualization, LABEL_SOURCE_AUTO); + + if (localhost->system_info->virt_detection) + label_list = + add_label_to_list(label_list, "_container", localhost->system_info->virt_detection, LABEL_SOURCE_AUTO); + + label_list = add_label_to_list( + label_list, "_is_master", (localhost->next || configured_as_master()) ? "true" : "false", LABEL_SOURCE_AUTO); + + if (localhost->rrdpush_send_destination) + label_list = + add_label_to_list(label_list, "_streams_to", localhost->rrdpush_send_destination, LABEL_SOURCE_AUTO); + + return label_list; +} + +struct label *load_config_labels() +{ + int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL); + if(!status) { + char *filename = CONFIG_DIR "/" CONFIG_FILENAME; + error("LABEL: Cannot reload the configuration file '%s', using labels in memory", filename); + } + + struct label *l = NULL; + struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL); + if(co) { + config_section_wrlock(co); + struct config_option *cv; + for(cv = co->values; cv ; cv = cv->next) { + char *name = cv->name; + if(is_valid_label_key(name) && strcmp(name, "from environment") && strcmp(name, "from kubernetes pods") ) { + l = add_label_to_list(l, name, cv->value, LABEL_SOURCE_NETDATA_CONF); + cv->flags |= CONFIG_VALUE_USED; + } else { + error("LABELS: It was not possible to create the label '%s' because it contains invalid character(s) or values." + , name); + } + } + config_section_unlock(co); + } + + return l; +} + +struct label *load_kubernetes_labels() +{ + struct label *l=NULL; + char *label_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)); + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh"); + if (unlikely(access(label_script, R_OK) != 0)) { + error("Kubernetes pod label fetching script %s not found.",label_script); + freez(label_script); + } else { + pid_t command_pid; + + debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script); + + FILE *fp = mypopen(label_script, &command_pid); + if(fp) { + int MAX_LINE_SIZE=300; + char buffer[MAX_LINE_SIZE + 1]; + while (fgets(buffer, MAX_LINE_SIZE, fp) != NULL) { + char *name=buffer; + char *value=buffer; + while (*value && *value != ':') value++; + if (*value == ':') { + *value = '\0'; + value++; + } + char *eos=value; + while (*eos && *eos != '\n') eos++; + if (*eos == '\n') *eos = '\0'; + if (strlen(value)>0) { + if (is_valid_label_key(name)){ + l = add_label_to_list(l, name, value, LABEL_SOURCE_KUBERNETES); + } else { + info("Ignoring invalid label name '%s'", name); + } + } else { + error("%s outputted unexpected result: '%s'", label_script, name); + } + }; + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':' + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null + int retcode=mypclose(fp, command_pid); + if (retcode) { + error("%s exited abnormally. No kubernetes labels will be added to the host.", label_script); + struct label *ll=l; + while (ll != NULL) { + info("Ignoring Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(ll->label_source), ll->key, ll->value); + ll = ll->next; + freez(l); + l=ll; + } + } + } + freez(label_script); + } + + return l; +} + +struct label *create_label(char *key, char *value, LABEL_SOURCE label_source) +{ + size_t key_len = strlen(key), value_len = strlen(value); + size_t n = sizeof(struct label) + key_len + 1 + value_len + 1; + struct label *result = callocz(1,n); + if (result != NULL) { + char *c = (char *)result; + c += sizeof(struct label); + strcpy(c, key); + result->key = c; + c += key_len + 1; + strcpy(c, value); + result->value = c; + result->label_source = label_source; + result->key_hash = simple_hash(result->key); + } + return result; +} + +struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source) +{ + struct label *lab = create_label(key, value, label_source); + lab->next = l; + return lab; +} + +int label_list_contains(struct label *head, struct label *check) +{ + while (head != NULL) + { + if (head->key_hash == check->key_hash && !strcmp(head->key, check->key)) + return 1; + head = head->next; + } + return 0; +} + +/* Create a list with entries from both lists. + If any entry in the low priority list is masked by an entry in the high priorty list then delete it. +*/ +struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri) +{ + struct label *result = hi_pri; + while (lo_pri != NULL) + { + struct label *current = lo_pri; + lo_pri = lo_pri->next; + if (!label_list_contains(result, current)) { + current->next = result; + result = current; + } + else + freez(current); + } + return result; +} + +void reload_host_labels() +{ |