Labels with dictionary (#13070)

* squashed and rebased to master

* fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h

* added unittest for UTF-8 multibyte sanitization

* Fix unit test compilation

* Fix CMake build

* remove double sanitizer for opentsdb; cleanup sanitize_json_string()

* rename error_description to error_message to avoid conflict with json-c

* revert last and undef error_description from json-c

* more unittests; attempt to fix protobuf map issue

* get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer

* added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted

* better sorted dictionary checking

* proper unittesting for sorted dictionaries

* call dictionary deletion callback when destroying the dictionary

* remove obsolete variable

* Fix exporting unit tests

* Fix k8s label parsing test

* workaround for cmocka and strdupz()

* Bypass cmocka memory allocation check

* Revert "Bypass cmocka memory allocation check"

This reverts commit 4c49923839d9229bea23ca914dd8a0be1ebe2bf4.

* Revert "workaround for cmocka and strdupz()"

This reverts commit 7bebee04801db1865c748a7896d5fa54bb7104a5.

* Bypass cmocka memory allocation checks

* respect json formatting for chart labels

* cloud sends colons

* print the value only once

* allow parenthesis in values and spaces; make stream sender send quotes for values

Co-authored-by: Vladimir Kobal <vlad@prokk.net>

61 files changed, 1965 insertions, 1398 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 378e8a6832..04b7c57427 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1412,6 +1412,7 @@ if(BUILD_TESTING)
         exporting/tests/exporting_doubles.c
         exporting/tests/netdata_doubles.c
         exporting/tests/system_doubles.c
+        database/rrdlabels.c
         )
     set(TEST_NAME exporting_engine)
     set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
@@ -1569,11 +1570,6 @@ endif()
         database/rrd.h
     )
     add_executable(cgroups_testdriver ${CGROUPS_TEST_FILES} ${CGROUPS_PLUGIN_FILES})
-    target_link_options(
-        cgroups_testdriver
-        PRIVATE
-        -Wl,--wrap=add_label_to_list
-    )
     target_link_libraries(cgroups_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
     add_test(NAME test_cgroups COMMAND cgroups_testdriver)
 
diff --git a/Makefile.am b/Makefile.am
index 6241fc1028..cdad1ea286 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1166,6 +1166,7 @@ if ENABLE_UNITTESTS
         $(EXPORTING_ENGINE_TEST_FILES) \
         $(EXPORTING_ENGINE_FILES) \
         $(LIBNETDATA_FILES) \
+        database/rrdlabels.c \
         $(NULL)
     exporting_tests_exporting_engine_testdriver_CFLAGS = \
         $(AM_CFLAGS) \
@@ -1259,7 +1260,7 @@ endif
         $(NULL)
     collectors_cgroups_plugin_tests_cgroups_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
     collectors_cgroups_plugin_tests_cgroups_testdriver_LDFLAGS = \
-        -Wl,--wrap=add_label_to_list \
+        -Wl,--wrap=rrdlabels_add \
         $(NULL)
 
 endif
diff --git a/aclk/aclk_api.c b/aclk/aclk_api.c
index a2e738ab1a..9f57f80638 100644
--- a/aclk/aclk_api.c
+++ b/aclk/aclk_api.c
@@ -45,13 +45,15 @@ void aclk_single_update_enable()
 }
 #endif /* ENABLE_ACLK */
 
-struct label *add_aclk_host_labels(struct label *label) {
+void add_aclk_host_labels(void) {
+    DICTIONARY *labels = localhost->host_labels;
+
 #ifdef ENABLE_ACLK
-    label = add_label_to_list(label, "_aclk_ng_available", "true", LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_aclk_ng_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
 #else
-    label = add_label_to_list(label, "_aclk_ng_available", "false", LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_aclk_ng_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
 #endif
-    label = add_label_to_list(label, "_aclk_legacy_available", "false", LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_aclk_legacy_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
 #ifdef ENABLE_ACLK
     ACLK_PROXY_TYPE aclk_proxy;
     char *proxy_str;
@@ -70,16 +72,16 @@ struct label *add_aclk_host_labels(struct label *label) {
     }
 
     int mqtt5 = config_get_boolean(CONFIG_SECTION_CLOUD, "mqtt5", CONFIG_BOOLEAN_NO);
-    label = add_label_to_list(label, "_mqtt_version", mqtt5 ? "5" : "3", LABEL_SOURCE_AUTO);
-    label = add_label_to_list(label, "_aclk_impl", "Next Generation", LABEL_SOURCE_AUTO);
-    label = add_label_to_list(label, "_aclk_proxy", proxy_str, LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_mqtt_version", mqtt5 ? "5" : "3", RRDLABEL_SRC_AUTO);
+    rrdlabels_add(labels, "_aclk_impl", "Next Generation", RRDLABEL_SRC_AUTO);
+    rrdlabels_add(labels, "_aclk_proxy", proxy_str, RRDLABEL_SRC_AUTO);
+
 #ifdef ENABLE_NEW_CLOUD_PROTOCOL
-    label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "true", LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
 #else
-    label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "false", LABEL_SOURCE_AUTO);
+    rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
 #endif
 #endif
-    return label;
 }
 
 char *aclk_state(void) {
diff --git a/aclk/aclk_api.h b/aclk/aclk_api.h
index 557b70d70a..b00a2a4e12 100644
--- a/aclk/aclk_api.h
+++ b/aclk/aclk_api.h
@@ -52,7 +52,7 @@ void aclk_host_state_update(RRDHOST *host, int connect);
 
 #endif
 
-struct label *add_aclk_host_labels(struct label *label);
+void add_aclk_host_labels(void);
 char *aclk_state(void);
 char *aclk_state_json(void);
 
diff --git a/aclk/schema-wrappers/chart_stream.cc b/aclk/schema-wrappers/chart_stream.cc
index 7d820e533b..54c9407586 100644
--- a/aclk/schema-wrappers/chart_stream.cc
+++ b/aclk/schema-wrappers/chart_stream.cc
@@ -76,7 +76,7 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance)
     freez((char*)instance->id);
     freez((char*)instance->claim_id);
 
-    free_label_list(instance->label_head);
+    rrdlabels_destroy(instance->chart_labels);
 
     freez((char*)instance->config_hash);
 }
@@ -85,7 +85,6 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
 {
     google::protobuf::Map<std::string, std::string> *map;
     aclk_lib::v1::ACLKMessagePosition *pos;
-    struct label *label;
 
     chart->set_id(update->id);
     chart->set_claim_id(update->claim_id);
@@ -93,11 +92,7 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
     chart->set_name(update->name);
 
     map = chart->mutable_chart_labels();
-    label = update->label_head;
-    while (label) {
-        map->insert({label->key, label->value});
-        label = label->next;
-    }
+    rrdlabels_walkthrough_read(update->chart_labels, label_add_to_map_callback, map);
 
     switch (update->memory_mode) {
     case RRD_MEMORY_MODE_NONE:
diff --git a/aclk/schema-wrappers/chart_stream.h b/aclk/schema-wrappers/chart_stream.h
index 7a46ecd8eb..9048668683 100644
--- a/aclk/schema-wrappers/chart_stream.h
+++ b/aclk/schema-wrappers/chart_stream.h
@@ -57,7 +57,7 @@ struct chart_instance_updated {
     const char *node_id;
     const char *name;
 
-    struct label *label_head;
+    DICTIONARY *chart_labels;
 
     RRD_MEMORY_MODE memory_mode;
 
diff --git a/aclk/schema-wrappers/node_info.cc b/aclk/schema-wrappers/node_info.cc
index f669852467..e0d6d34f01 100644
--- a/aclk/schema-wrappers/node_info.cc
+++ b/aclk/schema-wrappers/node_info.cc
@@ -6,7 +6,6 @@
 
 static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct aclk_node_info *data)
 {
-    struct label *label;
     google::protobuf::Map<std::string, std::string> *map;
 
     if (data->name)
@@ -67,12 +66,7 @@ static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct acl
     ml_info->set_ml_enabled(data->ml_info.ml_enabled);
 
     map = info->mutable_host_labels();
-    label = data->host_labels_head;
-    while (label) {
-        map->insert({label->key, label->value});
-        label = label->next;
-    }
-
+    rrdlabels_walkthrough_read(data->host_labels_ptr, label_add_to_map_callback, map);
     return 0;
 }
 
diff --git a/aclk/schema-wrappers/node_info.h b/aclk/schema-wrappers/node_info.h
index e67f3e1daa..d585f3f2d0 100644
--- a/aclk/schema-wrappers/node_info.h
+++ b/aclk/schema-wrappers/node_info.h
@@ -4,9 +4,10 @@
 #define ACLK_SCHEMA_WRAPPER_NODE_INFO_H
 
 #include <stdlib.h>
+#include <stdint.h>
 
-#include "database/rrd.h"
 #include "capability.h"
+#include "database/rrd.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -54,7 +55,7 @@ struct aclk_node_info {
 
     char *machine_guid;
 
-    struct label *host_labels_head;
+    DICTIONARY *host_labels_ptr;
 
     struct machine_learning_info ml_info;
 };
diff --git a/aclk/schema-wrappers/schema_wrapper_utils.cc b/aclk/schema-wrappers/schema_wrapper_utils.cc
index b100e20c3b..6573e62997 100644
--- a/aclk/schema-wrappers/schema_wrapper_utils.cc
+++ b/aclk/schema-wrappers/schema_wrapper_utils.cc
@@ -13,3 +13,10 @@ void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, st
     tv->tv_sec = ts.seconds();
     tv->tv_usec = ts.nanos()/1000;
 }
+
+int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    (void)ls;
+    auto map = (google::protobuf::Map<std::string, std::string> *)data;
+    map->insert({name, value});
+    return 1;
+}
diff --git a/aclk/schema-wrappers/schema_wrapper_utils.h b/aclk/schema-wrappers/schema_wrapper_utils.h
index 494855f822..2815d0f200 100644
--- a/aclk/schema-wrappers/schema_wrapper_utils.h
+++ b/aclk/schema-wrappers/schema_wrapper_utils.h
@@ -3,8 +3,11 @@
 #ifndef SCHEMA_WRAPPER_UTILS_H
 #define SCHEMA_WRAPPER_UTILS_H
 
+#include "database/rrd.h"
+
 #include <sys/time.h>
 #include <google/protobuf/timestamp.pb.h>
+#include <google/protobuf/map.h>
 
 #if GOOGLE_PROTOBUF_VERSION < 3001000
 #define PROTO_COMPAT_MSG_SIZE(msg) (size_t)msg.ByteSize();
@@ -16,5 +19,6 @@
 
 void set_google_timestamp_from_timeval(struct timeval tv, google::protobuf::Timestamp *ts);
 void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, struct timeval *tv);
+int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
 
 #endif /* SCHEMA_WRAPPER_UTILS_H */
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index 5676ef8ca1..9869f05015 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -776,7 +776,7 @@ struct cgroup {
 
     char *chart_title;
 
-    struct label *chart_labels;
+    DICTIONARY *chart_labels;
 
     struct cpuacct_stat cpuacct_stat;
     struct cpuacct_usage cpuacct_usage;
@@ -1735,34 +1735,24 @@ static inline void substitute_dots_in_id(char *s) {
     }
 }
 
-char *k8s_parse_resolved_name(struct label **labels, char *data) {
-    char *name = mystrsep(&data, " ");
-    
-    if (!data) {
-        return name;
-    }
-
-    while (data) {
-        char *key = mystrsep(&data, "=");
-
-        char *value;
-        if (data && *data == ',') {
-            value = "";
-            *data++ = '\0';
-        } else {
-            value = mystrsep(&data, ",");
-        }
-        value = strip_double_quotes(value, 1);
+// ----------------------------------------------------------------------------
+// parse k8s labels
 
-        if (!key || *key == '\0' || !value || *value == '\0')
-            continue;
+char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) {
+    // the first word, up to the first space is the name
+    char *name = mystrsep(&data, " ");
 
-        *labels = add_label_to_list(*labels, key, value, LABEL_SOURCE_KUBERNETES);
+    // the rest are key=value pairs separated by comma
+    while(data) {
+        char *pair = mystrsep(&data, ",");
+        rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S);
     }
 
     return name;
 }
 
+// ----------------------------------------------------------------------------
+
 static inline void free_pressure(struct pressure *res) {
     if (res->some.share_time.st)   rrdset_is_obsolete(res->some.share_time.st);
     if (res->some.total_time.st)   rrdset_is_obsolete(res->some.total_time.st);
@@ -1834,7 +1824,7 @@ static inline void cgroup_free(struct cgroup *cg) {
     freez(cg->chart_id);
     freez(cg->chart_title);
 
-    free_label_list(cg->chart_labels);
+    rrdlabels_destroy(cg->chart_labels);
 
     freez(cg);
 
@@ -1870,31 +1860,33 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {
         case 0:
             cg->pending_renames = 0;
             break;
+
         case 3:
             cg->pending_renames = 0;
             cg->processed = 1;
             break;
     }
 
-    if (cg->pending_renames || cg->processed) {
-        return;
-    }
-    if (!(new_name && *new_name && *new_name != '\n')) {
-        return;
-    }
-    new_name = trim(new_name);
-    if (!(new_name)) {
-        return;
-    }
+    if(cg->pending_renames || cg->processed) return;
+    if(!new_name || !*new_name || *new_name == '\n') return;
+    if(!(new_name = trim(new_name))) return;
+
     char *name = new_name;
     if (!strncmp(new_name, "k8s_", 4)) {
-        free_label_list(cg->chart_labels);
-        name = k8s_parse_resolved_name(&cg->chart_labels, new_name);
+        if(!cg->chart_labels) cg->chart_labels = rrdlabels_create();
+
+        // read the new labels and remove the obsolete ones
+        rrdlabels_unmark_all(cg->chart_labels);
+        name = k8s_parse_resolved_name_and_labels(cg->chart_labels, new_name);
+        rrdlabels_remove_all_unmarked(cg->chart_labels);
     }
+
     freez(cg->chart_title);
     cg->chart_title = cgroup_title_strdupz(name);
+
     freez(cg->chart_id);
     cg->chart_id = cgroup_chart_id_strdupz(name);
+
     substitute_dots_in_id(cg->chart_id);
     cg->hash_chart = simple_hash(cg->chart_id);
 }
@@ -3782,7 +3774,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_STACKED
                 );
 
-                rrdset_update_labels(cg->st_cpu, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_cpu, cg->chart_labels);
 
                 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
                     rrddim_add(cg->st_cpu, "user", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
@@ -3855,7 +3847,7 @@ void update_cgroup_charts(int update_every) {
                                     , RRDSET_TYPE_LINE
                             );
 
-                            rrdset_update_labels(cg->st_cpu_limit, cg->chart_labels);
+                            rrdset_update_rrdlabels(cg->st_cpu_limit, cg->chart_labels);
 
                             if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED))
                                 rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
@@ -3908,7 +3900,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_LINE
                 );
 
-                rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_cpu_nr_throttled, cg->chart_labels);
                 rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
             } else {
                 rrdset_next(cg->st_cpu_nr_throttled);
@@ -3934,7 +3926,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_LINE
                 );
 
-                rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_cpu_throttled_time, cg->chart_labels);
                 rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
             } else {
                 rrdset_next(cg->st_cpu_throttled_time);
@@ -3962,7 +3954,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_LINE
                 );
 
-                rrdset_update_labels(cg->st_cpu_shares, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_cpu_shares, cg->chart_labels);
                 rrddim_add(cg->st_cpu_shares, "shares", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
             } else {
                 rrdset_next(cg->st_cpu_shares);
@@ -3993,7 +3985,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_STACKED
                 );
 
-                rrdset_update_labels(cg->st_cpu_per_core, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_cpu_per_core, cg->chart_labels);
 
                 for(i = 0; i < cg->cpuacct_usage.cpus; i++) {
                     snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
@@ -4028,8 +4020,8 @@ void update_cgroup_charts(int update_every) {
                         , update_every
                         , RRDSET_TYPE_STACKED
                 );
-                
-                rrdset_update_labels(cg->st_mem, cg->chart_labels);
+
+                rrdset_update_rrdlabels(cg->st_mem, cg->chart_labels);
 
                 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
                     rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4089,7 +4081,7 @@ void update_cgroup_charts(int update_every) {
                         , RRDSET_TYPE_AREA
                 );
 
-                rrdset_update_labels(cg->st_writeback, cg->chart_labels);
+                rrdset_update_rrdlabels(cg->st_writeback, cg->chart_labels);
 
                 if(cg->memory.detailed_has_dirty)
                     rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4124,7 +4116,7 @@ void update_cgroup_charts(int update_every) {
                             , RRDSET_TYPE_LINE
                     );
 
-                    rrdset_update_labels(cg->st_mem_activity, cg->chart_labels);
+                    rrdset_update_rrdlabels(cg->st_mem_activity, cg->chart_labels);
 
                     rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
                     rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4155,7 +41