Labels issues (#7515)

Initial work on host labels from the dedicated branch. Includes work for issues #7096, #7400, #7411, #7369, #7410, #7458, #7459, #7412 and #7408 by @vlvkobal, @thiagoftsm, @cakrit and @amoss.

7 files changed, 424 insertions, 6 deletions

diff --git a/database/rrd.h b/database/rrd.h
index c75b0deebe..432341d75d 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -148,6 +148,25 @@ typedef enum rrddim_flags {
 #define rrddim_flag_clear(rd, flag) (rd)->flags &= ~(flag)
 #endif
 
+typedef enum label_source {
+    LABEL_SOURCE_AUTO             = 0,
+    LABEL_SOURCE_NETDATA_CONF     = 1,
+    LABEL_SOURCE_DOCKER           = 2,
+    LABEL_SOURCE_ENVIRONMENT      = 3,
+    LABEL_SOURCE_KUBERNETES       = 4
+} LABEL_SOURCE;
+
+struct label {
+    char *key, *value;
+    uint32_t key_hash;
+    LABEL_SOURCE label_source;
+    struct label *next;
+};
+
+char *translate_label_source(LABEL_SOURCE l);
+struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
+struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
+void reload_host_labels();
 
 // ----------------------------------------------------------------------------
 // RRD DIMENSION - this is a metric
@@ -727,6 +746,11 @@ struct rrdhost {
     netdata_rwlock_t rrdhost_rwlock;                // lock for this RRDHOST (protects rrdset_root linked list)
 
     // ------------------------------------------------------------------------
+    // Support for host-level labels
+    struct label *labels;
+    netdata_rwlock_t labels_rwlock;         // lock for the label list
+
+    // ------------------------------------------------------------------------
     // indexes
 
     avl_tree_lock rrdset_root_index;                // the host's charts index (by id)
diff --git a/database/rrdcalc.c b/database/rrdcalc.c
index 9f16ce3740..a0a7f51141 100644
--- a/database/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -559,6 +559,8 @@ void rrdcalc_free(RRDCALC *rc) {
     freez(rc->units);
     freez(rc->info);
     simple_pattern_free(rc->spdim);
+    freez(rc->labels);
+    simple_pattern_free(rc->splabels);
     freez(rc);
 }
 
@@ -603,6 +605,91 @@ void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
     rrdcalc_free(rc);
 }
 
+void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
+
+    if(unlikely(rc == host->alarms_with_foreach))
+        host->alarms_with_foreach = rc->next;
+    else {
+        RRDCALC *t;
+        for(t = host->alarms_with_foreach; t && t->next != rc; t = t->next) ;
+        if(t) {
+            t->next = rc->next;
+            rc->next = NULL;
+        }
+        else
+            error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
+    }
+
+    rrdcalc_free(rc);
+}
+
+static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) {
+    RRDCALC *rc = alarms;
+    while (rc) {
+        if (!rc->labels) {
+            rc = rc->next;
+            continue;
+        }
+
+        char cmp[CONFIG_FILE_LINE_MAX+1];
+        struct label *move = host->labels;
+        while(move) {
+            snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
+            if (simple_pattern_matches(rc->splabels, move->key) ||
+                simple_pattern_matches(rc->splabels, cmp)) {
+                break;
+            }
+
+            move = move->next;
+        }
+
+        RRDCALC *next = rc->next;
+        if(!move) {
+            info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
+                 rc->name,
+                 host->hostname,
+                 rc->labels);
+
+            if(host->alarms == alarms) {
+                rrdcalc_unlink_and_free(host, rc);
+            } else
+                rrdcalc_foreach_unlink_and_free(host, rc);
+
+        }
+
+        rc = next;
+    }
+}
+
+void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) {
+    netdata_rwlock_rdlock(&host->labels_rwlock);
+
+    rrdcalc_labels_unlink_alarm_loop(host, host->alarms);
+    rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach);
+
+    netdata_rwlock_unlock(&host->labels_rwlock);
+}
+
+void rrdcalc_labels_unlink() {
+    rrd_rdlock();
+
+    RRDHOST *host;
+    rrdhost_foreach_read(host) {
+        if (unlikely(!host->health_enabled))
+            continue;
+
+        if (host->labels) {
+            rrdhost_wrlock(host);
+
+            rrdcalc_labels_unlink_alarm_from_host(host);
+
+            rrdhost_unlock(host);
+        }
+    }
+
+    rrd_unlock();
+}
+
 // ----------------------------------------------------------------------------
 // Alarm
 
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
index e0b6325971..e47bba1cd6 100644
--- a/database/rrdcalc.h
+++ b/database/rrdcalc.h
@@ -91,6 +91,11 @@ struct rrdcalc {
     uint32_t crit_repeat_every; // interval between repeating critical notifications
 
     // ------------------------------------------------------------------------
+    // Labels settings
+    char *labels;                   // the label read from an alarm file
+    SIMPLE_PATTERN *splabels;       // the simple pattern of labels
+
+    // ------------------------------------------------------------------------
     // runtime information
 
     RRDCALC_STATUS old_status; // the old status of the alarm
@@ -157,6 +162,9 @@ extern void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc);
 extern void dimension_remove_pipe_comma(char *str);
 extern char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen);
 
+extern void rrdcalc_labels_unlink();
+extern void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host);
+
 static inline int rrdcalc_isrepeating(RRDCALC *rc) {
     if (unlikely(rc->warn_repeat_every > 0 || rc->crit_repeat_every > 0)) {
         return 1;
diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c
index f7a0855611..58ec95a087 100644
--- a/database/rrdcalctemplate.c
+++ b/database/rrdcalctemplate.c
@@ -4,6 +4,45 @@
 #include "rrd.h"
 
 // ----------------------------------------------------------------------------
+
+static int rrdcalctemplate_is_there_label_restriction(RRDCALCTEMPLATE *rt,  RRDHOST *host) {
+    if(!rt->labels)
+        return 0;
+
+    errno = 0;
+    struct label *move = host->labels;
+    char cmp[CONFIG_FILE_LINE_MAX+1];
+
+    int ret;
+    if(move) {
+        netdata_rwlock_rdlock(&host->labels_rwlock);
+        while(move) {
+            snprintfz(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
+            if (simple_pattern_matches(rt->splabels, move->key) ||
+                simple_pattern_matches(rt->splabels, cmp)) {
+                break;
+            }
+            move = move->next;
+        }
+        netdata_rwlock_unlock(&host->labels_rwlock);
+
+        if(!move) {
+            error("Health template '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
+                   rt->name,
+                   host->hostname,
+                   rt->labels
+            );
+            ret = 1;
+        } else {
+            ret = 0;
+        }
+    } else {
+        ret =0;
+    }
+
+    return ret;
+}
+
 // RRDCALCTEMPLATE management
 /**
  * RRDCALC TEMPLATE LINK MATCHING
@@ -14,13 +53,18 @@
 void rrdcalctemplate_link_matching_test(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host ) {
     if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)
        && (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) {
-        RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
-        if(unlikely(!rc))
-            info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname);
+        if (!rrdcalctemplate_is_there_label_restriction(rt, host)) {
+            RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
+            if (unlikely(!rc))
+                info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed",
+                     rt->name, st->id, host->hostname);
 #ifdef NETDATA_INTERNAL_CHECKS
-        else if(rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
-            error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id);
+            else if (rc->rrdset != st &&
+                     !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
+                error("Health alarm '%s.%s' should be linked to chart '%s', but it is not",
+                      rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
 #endif
+        }
     }
 }
 
@@ -56,7 +100,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
     freez(rt->info);
     freez(rt->dimensions);
     freez(rt->foreachdim);
+    freez(rt->labels);
     simple_pattern_free(rt->spdim);
+    simple_pattern_free(rt->splabels);
     freez(rt);
 }
 
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 676b4cf645..9cfd09846e 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -59,6 +59,11 @@ struct rrdcalctemplate {
     uint32_t crit_repeat_every; // interval between repeating critical notifications
 
     // ------------------------------------------------------------------------
+    // Labels settings
+    char *labels;                   // the label read from an alarm file
+    SIMPLE_PATTERN *splabels;       // the simple pattern of labels
+
+    // ------------------------------------------------------------------------
     // expressions related to the alarm
 
     EVAL_EXPRESSION *calculation;
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 737e066c9d..150596671f 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -709,6 +709,248 @@ void rrdhost_save_charts(RRDHOST *host) {
     rrdhost_unlock(host);
 }
 
+static int is_valid_label_key(char *key) {
+    //Prometheus exporter
+    if(!strcmp(key, "chart") || !strcmp(key, "family")  || !strcmp(key, "dimension"))
+        return 0;
+
+    //Netdata and Prometheus  internal
+    if (*key == '_')
+        return 0;
+
+    while(*key) {
+        if(!(isdigit(*key) || isalpha(*key) || *key == '.' || *key == '_' || *key == '-'))
+            return 0;
+
+        key++;
+    }
+
+    return 1;
+}
+
+char *translate_label_source(LABEL_SOURCE l) {
+    switch (l) {
+        case LABEL_SOURCE_AUTO:
+            return "AUTO";
+        case LABEL_SOURCE_NETDATA_CONF:
+            return "NETDATA.CONF";
+        case LABEL_SOURCE_DOCKER :
+            return "DOCKER";
+        case LABEL_SOURCE_ENVIRONMENT  :
+            return "ENVIRONMENT";
+        case LABEL_SOURCE_KUBERNETES :
+            return "KUBERNETES";
+        default:
+            return "Invalid label source";
+    }
+}
+
+struct label *load_auto_labels()
+{
+    struct label *label_list = NULL;
+
+    if (localhost->system_info->os_name)
+        label_list =
+            add_label_to_list(label_list, "_os_name", localhost->system_info->os_name, LABEL_SOURCE_AUTO);
+
+    if (localhost->system_info->os_version)
+        label_list =
+            add_label_to_list(label_list, "_os_version", localhost->system_info->os_version, LABEL_SOURCE_AUTO);
+
+    if (localhost->system_info->kernel_version)
+        label_list =
+            add_label_to_list(label_list, "_kernel_version", localhost->system_info->kernel_version, LABEL_SOURCE_AUTO);
+
+    if (localhost->system_info->architecture)
+        label_list =
+            add_label_to_list(label_list, "_architecture", localhost->system_info->architecture, LABEL_SOURCE_AUTO);
+
+    if (localhost->system_info->virtualization)
+        label_list =
+            add_label_to_list(label_list, "_virtualization", localhost->system_info->virtualization, LABEL_SOURCE_AUTO);
+
+    if (localhost->system_info->virt_detection)
+        label_list =
+            add_label_to_list(label_list, "_container", localhost->system_info->virt_detection, LABEL_SOURCE_AUTO);
+
+    label_list = add_label_to_list(
+        label_list, "_is_master", (localhost->next || configured_as_master()) ? "true" : "false", LABEL_SOURCE_AUTO);
+
+    if (localhost->rrdpush_send_destination)
+        label_list =
+            add_label_to_list(label_list, "_streams_to", localhost->rrdpush_send_destination, LABEL_SOURCE_AUTO);
+
+    return label_list;
+}
+
+struct label *load_config_labels()
+{
+    int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
+    if(!status) {
+        char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
+        error("LABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
+    }
+
+    struct label *l = NULL;
+    struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
+    if(co) {
+        config_section_wrlock(co);
+        struct config_option *cv;
+        for(cv = co->values; cv ; cv = cv->next) {
+            char *name = cv->name;
+            if(is_valid_label_key(name) && strcmp(name, "from environment") && strcmp(name, "from kubernetes pods") ) {
+                l = add_label_to_list(l, name, cv->value, LABEL_SOURCE_NETDATA_CONF);
+                cv->flags |= CONFIG_VALUE_USED;
+            } else {
+                error("LABELS: It was not possible to create the label '%s' because it contains invalid character(s) or values."
+                       , name);
+            }
+        }
+        config_section_unlock(co);
+    }
+
+    return l;
+}
+
+struct label *load_kubernetes_labels()
+{
+    struct label *l=NULL;
+    char *label_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2));
+    sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
+    if (unlikely(access(label_script, R_OK) != 0)) {
+        error("Kubernetes pod label fetching script %s not found.",label_script);
+        freez(label_script);
+    } else {
+        pid_t command_pid;
+
+        debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
+
+        FILE *fp = mypopen(label_script, &command_pid);
+        if(fp) {
+            int MAX_LINE_SIZE=300;
+            char buffer[MAX_LINE_SIZE + 1];
+            while (fgets(buffer, MAX_LINE_SIZE, fp) != NULL) {
+                char *name=buffer;
+                char *value=buffer;
+                while (*value && *value != ':') value++;
+                if (*value == ':') {
+                    *value = '\0';
+                    value++;
+                }
+                char *eos=value;
+                while (*eos && *eos != '\n') eos++;
+                if (*eos == '\n') *eos = '\0';
+                if (strlen(value)>0) {
+                    if (is_valid_label_key(name)){
+                        l = add_label_to_list(l, name, value, LABEL_SOURCE_KUBERNETES);
+                    } else {
+                        info("Ignoring invalid label name '%s'", name);
+                    }
+                } else {
+                    error("%s outputted unexpected result: '%s'", label_script, name);
+                }
+            };
+            // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
+            // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
+            int retcode=mypclose(fp, command_pid);
+            if (retcode) {
+                error("%s exited abnormally. No kubernetes labels will be added to the host.", label_script);
+                struct label *ll=l;
+                while (ll != NULL) {
+                    info("Ignoring Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(ll->label_source), ll->key, ll->value);
+                    ll = ll->next;
+                    freez(l);
+                    l=ll;
+                }
+            }
+        }
+        freez(label_script);
+    }
+
+    return l;
+}
+
+struct label *create_label(char *key, char *value, LABEL_SOURCE label_source)
+{
+    size_t key_len = strlen(key), value_len = strlen(value);
+    size_t n = sizeof(struct label) + key_len + 1 + value_len + 1;
+    struct label *result = callocz(1,n);
+    if (result != NULL) {
+        char *c = (char *)result;
+        c += sizeof(struct label);
+        strcpy(c, key);
+        result->key = c;
+        c += key_len + 1;
+        strcpy(c, value);
+        result->value = c;
+        result->label_source = label_source;
+        result->key_hash = simple_hash(result->key);
+    }
+    return result;
+}
+
+struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source)
+{
+    struct label *lab = create_label(key, value, label_source);
+    lab->next = l;
+    return lab;
+}
+
+int label_list_contains(struct label *head, struct label *check)
+{
+    while (head != NULL)
+    {
+        if (head->key_hash == check->key_hash && !strcmp(head->key, check->key))
+            return 1;
+        head = head->next;
+    }
+    return 0;
+}
+
+/* Create a list with entries from both lists.
+   If any entry in the low priority list is masked by an entry in the high priorty list then delete it.
+*/
+struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri)
+{
+    struct label *result = hi_pri;
+    while (lo_pri != NULL)
+    {
+        struct label *current = lo_pri;
+        lo_pri = lo_pri->next;
+        if (!label_list_contains(result, current)) {
+            current->next = result;
+            result = current;
+        }
+        else
+            freez(current);
+    }
+    return result;
+}
+
+void reload_host_labels()
+{
+    struct label *from_auto = load_auto_labels();
+    struct label *from_k8s = load_kubernetes_labels();
+    struct label *from_config = load_config_labels();
+
+    struct label *new_labels = merge_label_lists(from_auto, from_k8s);
+    new_labels = merge_label_lists(new_labels, from_config);
+
+    netdata_rwlock_wrlock(&localhost->labels_rwlock);
+    struct label *old_labels = localhost->labels;
+    localhost->labels = new_labels;
+    netdata_rwlock_unlock(&localhost->labels_rwlock);
+
+    while (old_labels != NULL)
+    {
+        struct label *current = old_labels;
+        old_labels = old_labels->next;
+        freez(current);
+    }
+
+    health_reload();
+}
+
 // ----------------------------------------------------------------------------
 // RRDHOST - delete host files
 
diff --git a/database/rrdvar.c b/database/rrdvar.c
index 0858d0bdcd..d819a08afd 100644
--- a/database/rrdvar.c
+++ b/database/rrdvar.c
@@ -291,12 +291,18 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) {
 
     buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", st->id, st->name, st->context);
     avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper);
+
     buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", st->family);
     helper.counter = 0;
     avl_traverse_lock(&st->rrdfamily->rrdvar_root_index, single_variable2json, (void *)&helper);
-    buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", host->hostname);
+
+    buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",", host->hostname);
+    buffer_strcat(buf, "\n\t\"labels\": {\n");
+    host_labels2json(st->rrdhost, buf, 2);
+    buffer_strcat(buf, "\t},\n\t\"host_variables\": {");
     helper.counter = 0;
     avl_traverse_lock(&host->rrdvar_root_index, single_variable2json, (void *)&helper);
+
     buffer_strcat(buf, "\n\t}\n}\n");
 }