From 1b0f6c6b2296dc082d85f38c298a61442dcf2490 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 13 Jun 2022 20:35:45 +0300 Subject: Labels with dictionary (#13070) * squashed and rebased to master * fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h * added unittest for UTF-8 multibyte sanitization * Fix unit test compilation * Fix CMake build * remove double sanitizer for opentsdb; cleanup sanitize_json_string() * rename error_description to error_message to avoid conflict with json-c * revert last and undef error_description from json-c * more unittests; attempt to fix protobuf map issue * get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer * added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted * better sorted dictionary checking * proper unittesting for sorted dictionaries * call dictionary deletion callback when destroying the dictionary * remove obsolete variable * Fix exporting unit tests * Fix k8s label parsing test * workaround for cmocka and strdupz() * Bypass cmocka memory allocation check * Revert "Bypass cmocka memory allocation check" This reverts commit 4c49923839d9229bea23ca914dd8a0be1ebe2bf4. * Revert "workaround for cmocka and strdupz()" This reverts commit 7bebee04801db1865c748a7896d5fa54bb7104a5. * Bypass cmocka memory allocation checks * respect json formatting for chart labels * cloud sends colons * print the value only once * allow parenthesis in values and spaces; make stream sender send quotes for values Co-authored-by: Vladimir Kobal --- CMakeLists.txt | 6 +- Makefile.am | 3 +- aclk/aclk_api.c | 22 +- aclk/aclk_api.h | 2 +- aclk/schema-wrappers/chart_stream.cc | 9 +- aclk/schema-wrappers/chart_stream.h | 2 +- aclk/schema-wrappers/node_info.cc | 8 +- aclk/schema-wrappers/node_info.h | 5 +- aclk/schema-wrappers/schema_wrapper_utils.cc | 7 + aclk/schema-wrappers/schema_wrapper_utils.h | 4 + collectors/cgroups.plugin/sys_fs_cgroup.c | 142 ++- collectors/cgroups.plugin/sys_fs_cgroup.h | 2 +- .../cgroups.plugin/tests/test_cgroups_plugin.c | 87 +- collectors/cgroups.plugin/tests/test_doubles.c | 26 +- collectors/plugins.d/pluginsd_parser.c | 61 +- collectors/plugins.d/pluginsd_parser.h | 12 +- collectors/proc.plugin/plugin_proc.h | 2 +- collectors/proc.plugin/proc_net_dev.c | 44 +- daemon/commands.c | 12 +- daemon/get-kubernetes-labels.sh.in | 17 +- daemon/main.c | 3 + database/rrd.h | 117 +- database/rrdcalc.c | 91 +- database/rrdcalc.h | 4 +- database/rrdcalctemplate.c | 102 +- database/rrdcalctemplate.h | 4 +- database/rrdhost.c | 278 +---- database/rrdlabels.c | 1230 +++++++++++++++++--- database/rrdset.c | 105 +- database/sqlite/sqlite_aclk_chart.c | 12 +- database/sqlite/sqlite_aclk_node.c | 5 +- exporting/check_filters.c | 8 + exporting/clean_connectors.c | 2 +- exporting/exporting_engine.h | 9 +- exporting/graphite/graphite.c | 30 +- exporting/graphite/graphite.h | 2 +- exporting/json/json.c | 34 +- exporting/opentsdb/opentsdb.c | 70 +- exporting/opentsdb/opentsdb.h | 2 +- exporting/process_data.c | 4 +- exporting/prometheus/prometheus.c | 79 +- exporting/prometheus/remote_write/remote_write.c | 40 +- exporting/tests/exporting_fixtures.c | 23 +- exporting/tests/netdata_doubles.c | 8 + exporting/tests/test_exporting_engine.c | 22 +- exporting/tests/test_exporting_engine.h | 7 + health/health_config.c | 26 +- health/health_log.c | 21 +- libnetdata/config/appconfig.c | 12 + libnetdata/dictionary/dictionary.c | 323 ++--- libnetdata/dictionary/dictionary.h | 12 +- libnetdata/inlined.h | 28 +- libnetdata/json/json.h | 9 +- parser/parser.h | 8 +- registry/registry_internals.h | 2 +- streaming/receiver.c | 9 +- streaming/rrdpush.c | 48 +- streaming/sender.c | 8 +- web/api/formatters/json_wrapper.c | 41 +- web/api/formatters/rrdset2json.c | 18 +- web/api/web_api_v1.c | 34 +- 61 files changed, 1965 insertions(+), 1398 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 378e8a6832..04b7c57427 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1412,6 +1412,7 @@ if(BUILD_TESTING) exporting/tests/exporting_doubles.c exporting/tests/netdata_doubles.c exporting/tests/system_doubles.c + database/rrdlabels.c ) set(TEST_NAME exporting_engine) set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS) @@ -1569,11 +1570,6 @@ endif() database/rrd.h ) add_executable(cgroups_testdriver ${CGROUPS_TEST_FILES} ${CGROUPS_PLUGIN_FILES}) - target_link_options( - cgroups_testdriver - PRIVATE - -Wl,--wrap=add_label_to_list - ) target_link_libraries(cgroups_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES}) add_test(NAME test_cgroups COMMAND cgroups_testdriver) diff --git a/Makefile.am b/Makefile.am index 6241fc1028..cdad1ea286 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1166,6 +1166,7 @@ if ENABLE_UNITTESTS $(EXPORTING_ENGINE_TEST_FILES) \ $(EXPORTING_ENGINE_FILES) \ $(LIBNETDATA_FILES) \ + database/rrdlabels.c \ $(NULL) exporting_tests_exporting_engine_testdriver_CFLAGS = \ $(AM_CFLAGS) \ @@ -1259,7 +1260,7 @@ endif $(NULL) collectors_cgroups_plugin_tests_cgroups_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS) collectors_cgroups_plugin_tests_cgroups_testdriver_LDFLAGS = \ - -Wl,--wrap=add_label_to_list \ + -Wl,--wrap=rrdlabels_add \ $(NULL) endif diff --git a/aclk/aclk_api.c b/aclk/aclk_api.c index a2e738ab1a..9f57f80638 100644 --- a/aclk/aclk_api.c +++ b/aclk/aclk_api.c @@ -45,13 +45,15 @@ void aclk_single_update_enable() } #endif /* ENABLE_ACLK */ -struct label *add_aclk_host_labels(struct label *label) { +void add_aclk_host_labels(void) { + DICTIONARY *labels = localhost->host_labels; + #ifdef ENABLE_ACLK - label = add_label_to_list(label, "_aclk_ng_available", "true", LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_aclk_ng_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); #else - label = add_label_to_list(label, "_aclk_ng_available", "false", LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_aclk_ng_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); #endif - label = add_label_to_list(label, "_aclk_legacy_available", "false", LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_aclk_legacy_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); #ifdef ENABLE_ACLK ACLK_PROXY_TYPE aclk_proxy; char *proxy_str; @@ -70,16 +72,16 @@ struct label *add_aclk_host_labels(struct label *label) { } int mqtt5 = config_get_boolean(CONFIG_SECTION_CLOUD, "mqtt5", CONFIG_BOOLEAN_NO); - label = add_label_to_list(label, "_mqtt_version", mqtt5 ? "5" : "3", LABEL_SOURCE_AUTO); - label = add_label_to_list(label, "_aclk_impl", "Next Generation", LABEL_SOURCE_AUTO); - label = add_label_to_list(label, "_aclk_proxy", proxy_str, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_mqtt_version", mqtt5 ? "5" : "3", RRDLABEL_SRC_AUTO); + rrdlabels_add(labels, "_aclk_impl", "Next Generation", RRDLABEL_SRC_AUTO); + rrdlabels_add(labels, "_aclk_proxy", proxy_str, RRDLABEL_SRC_AUTO); + #ifdef ENABLE_NEW_CLOUD_PROTOCOL - label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "true", LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); #else - label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "false", LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); #endif #endif - return label; } char *aclk_state(void) { diff --git a/aclk/aclk_api.h b/aclk/aclk_api.h index 557b70d70a..b00a2a4e12 100644 --- a/aclk/aclk_api.h +++ b/aclk/aclk_api.h @@ -52,7 +52,7 @@ void aclk_host_state_update(RRDHOST *host, int connect); #endif -struct label *add_aclk_host_labels(struct label *label); +void add_aclk_host_labels(void); char *aclk_state(void); char *aclk_state_json(void); diff --git a/aclk/schema-wrappers/chart_stream.cc b/aclk/schema-wrappers/chart_stream.cc index 7d820e533b..54c9407586 100644 --- a/aclk/schema-wrappers/chart_stream.cc +++ b/aclk/schema-wrappers/chart_stream.cc @@ -76,7 +76,7 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance) freez((char*)instance->id); freez((char*)instance->claim_id); - free_label_list(instance->label_head); + rrdlabels_destroy(instance->chart_labels); freez((char*)instance->config_hash); } @@ -85,7 +85,6 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co { google::protobuf::Map *map; aclk_lib::v1::ACLKMessagePosition *pos; - struct label *label; chart->set_id(update->id); chart->set_claim_id(update->claim_id); @@ -93,11 +92,7 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co chart->set_name(update->name); map = chart->mutable_chart_labels(); - label = update->label_head; - while (label) { - map->insert({label->key, label->value}); - label = label->next; - } + rrdlabels_walkthrough_read(update->chart_labels, label_add_to_map_callback, map); switch (update->memory_mode) { case RRD_MEMORY_MODE_NONE: diff --git a/aclk/schema-wrappers/chart_stream.h b/aclk/schema-wrappers/chart_stream.h index 7a46ecd8eb..9048668683 100644 --- a/aclk/schema-wrappers/chart_stream.h +++ b/aclk/schema-wrappers/chart_stream.h @@ -57,7 +57,7 @@ struct chart_instance_updated { const char *node_id; const char *name; - struct label *label_head; + DICTIONARY *chart_labels; RRD_MEMORY_MODE memory_mode; diff --git a/aclk/schema-wrappers/node_info.cc b/aclk/schema-wrappers/node_info.cc index f669852467..e0d6d34f01 100644 --- a/aclk/schema-wrappers/node_info.cc +++ b/aclk/schema-wrappers/node_info.cc @@ -6,7 +6,6 @@ static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct aclk_node_info *data) { - struct label *label; google::protobuf::Map *map; if (data->name) @@ -67,12 +66,7 @@ static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct acl ml_info->set_ml_enabled(data->ml_info.ml_enabled); map = info->mutable_host_labels(); - label = data->host_labels_head; - while (label) { - map->insert({label->key, label->value}); - label = label->next; - } - + rrdlabels_walkthrough_read(data->host_labels_ptr, label_add_to_map_callback, map); return 0; } diff --git a/aclk/schema-wrappers/node_info.h b/aclk/schema-wrappers/node_info.h index e67f3e1daa..d585f3f2d0 100644 --- a/aclk/schema-wrappers/node_info.h +++ b/aclk/schema-wrappers/node_info.h @@ -4,9 +4,10 @@ #define ACLK_SCHEMA_WRAPPER_NODE_INFO_H #include +#include -#include "database/rrd.h" #include "capability.h" +#include "database/rrd.h" #ifdef __cplusplus extern "C" { @@ -54,7 +55,7 @@ struct aclk_node_info { char *machine_guid; - struct label *host_labels_head; + DICTIONARY *host_labels_ptr; struct machine_learning_info ml_info; }; diff --git a/aclk/schema-wrappers/schema_wrapper_utils.cc b/aclk/schema-wrappers/schema_wrapper_utils.cc index b100e20c3b..6573e62997 100644 --- a/aclk/schema-wrappers/schema_wrapper_utils.cc +++ b/aclk/schema-wrappers/schema_wrapper_utils.cc @@ -13,3 +13,10 @@ void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, st tv->tv_sec = ts.seconds(); tv->tv_usec = ts.nanos()/1000; } + +int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + (void)ls; + auto map = (google::protobuf::Map *)data; + map->insert({name, value}); + return 1; +} diff --git a/aclk/schema-wrappers/schema_wrapper_utils.h b/aclk/schema-wrappers/schema_wrapper_utils.h index 494855f822..2815d0f200 100644 --- a/aclk/schema-wrappers/schema_wrapper_utils.h +++ b/aclk/schema-wrappers/schema_wrapper_utils.h @@ -3,8 +3,11 @@ #ifndef SCHEMA_WRAPPER_UTILS_H #define SCHEMA_WRAPPER_UTILS_H +#include "database/rrd.h" + #include #include +#include #if GOOGLE_PROTOBUF_VERSION < 3001000 #define PROTO_COMPAT_MSG_SIZE(msg) (size_t)msg.ByteSize(); @@ -16,5 +19,6 @@ void set_google_timestamp_from_timeval(struct timeval tv, google::protobuf::Timestamp *ts); void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, struct timeval *tv); +int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data); #endif /* SCHEMA_WRAPPER_UTILS_H */ diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c index 5676ef8ca1..9869f05015 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -776,7 +776,7 @@ struct cgroup { char *chart_title; - struct label *chart_labels; + DICTIONARY *chart_labels; struct cpuacct_stat cpuacct_stat; struct cpuacct_usage cpuacct_usage; @@ -1735,34 +1735,24 @@ static inline void substitute_dots_in_id(char *s) { } } -char *k8s_parse_resolved_name(struct label **labels, char *data) { - char *name = mystrsep(&data, " "); - - if (!data) { - return name; - } - - while (data) { - char *key = mystrsep(&data, "="); - - char *value; - if (data && *data == ',') { - value = ""; - *data++ = '\0'; - } else { - value = mystrsep(&data, ","); - } - value = strip_double_quotes(value, 1); +// ---------------------------------------------------------------------------- +// parse k8s labels - if (!key || *key == '\0' || !value || *value == '\0') - continue; +char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) { + // the first word, up to the first space is the name + char *name = mystrsep(&data, " "); - *labels = add_label_to_list(*labels, key, value, LABEL_SOURCE_KUBERNETES); + // the rest are key=value pairs separated by comma + while(data) { + char *pair = mystrsep(&data, ","); + rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S); } return name; } +// ---------------------------------------------------------------------------- + static inline void free_pressure(struct pressure *res) { if (res->some.share_time.st) rrdset_is_obsolete(res->some.share_time.st); if (res->some.total_time.st) rrdset_is_obsolete(res->some.total_time.st); @@ -1834,7 +1824,7 @@ static inline void cgroup_free(struct cgroup *cg) { freez(cg->chart_id); freez(cg->chart_title); - free_label_list(cg->chart_labels); + rrdlabels_destroy(cg->chart_labels); freez(cg); @@ -1870,31 +1860,33 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) { case 0: cg->pending_renames = 0; break; + case 3: cg->pending_renames = 0; cg->processed = 1; break; } - if (cg->pending_renames || cg->processed) { - return; - } - if (!(new_name && *new_name && *new_name != '\n')) { - return; - } - new_name = trim(new_name); - if (!(new_name)) { - return; - } + if(cg->pending_renames || cg->processed) return; + if(!new_name || !*new_name || *new_name == '\n') return; + if(!(new_name = trim(new_name))) return; + char *name = new_name; if (!strncmp(new_name, "k8s_", 4)) { - free_label_list(cg->chart_labels); - name = k8s_parse_resolved_name(&cg->chart_labels, new_name); + if(!cg->chart_labels) cg->chart_labels = rrdlabels_create(); + + // read the new labels and remove the obsolete ones + rrdlabels_unmark_all(cg->chart_labels); + name = k8s_parse_resolved_name_and_labels(cg->chart_labels, new_name); + rrdlabels_remove_all_unmarked(cg->chart_labels); } + freez(cg->chart_title); cg->chart_title = cgroup_title_strdupz(name); + freez(cg->chart_id); cg->chart_id = cgroup_chart_id_strdupz(name); + substitute_dots_in_id(cg->chart_id); cg->hash_chart = simple_hash(cg->chart_id); } @@ -3782,7 +3774,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_STACKED ); - rrdset_update_labels(cg->st_cpu, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu, cg->chart_labels); if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) { rrddim_add(cg->st_cpu, "user", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL); @@ -3855,7 +3847,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_cpu_limit, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu_limit, cg->chart_labels); if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE); @@ -3908,7 +3900,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu_nr_throttled, cg->chart_labels); rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else { rrdset_next(cg->st_cpu_nr_throttled); @@ -3934,7 +3926,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu_throttled_time, cg->chart_labels); rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(cg->st_cpu_throttled_time); @@ -3962,7 +3954,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_cpu_shares, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu_shares, cg->chart_labels); rrddim_add(cg->st_cpu_shares, "shares", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else { rrdset_next(cg->st_cpu_shares); @@ -3993,7 +3985,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_STACKED ); - rrdset_update_labels(cg->st_cpu_per_core, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_cpu_per_core, cg->chart_labels); for(i = 0; i < cg->cpuacct_usage.cpus; i++) { snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i); @@ -4028,8 +4020,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_STACKED ); - - rrdset_update_labels(cg->st_mem, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_mem, cg->chart_labels); if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) { rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); @@ -4089,7 +4081,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_AREA ); - rrdset_update_labels(cg->st_writeback, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_writeback, cg->chart_labels); if(cg->memory.detailed_has_dirty) rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); @@ -4124,7 +4116,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_mem_activity, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_mem_activity, cg->chart_labels); rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); @@ -4155,7 +4147,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_pgfaults, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_pgfaults, cg->chart_labels); rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); @@ -4187,7 +4179,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_STACKED ); - rrdset_update_labels(cg->st_mem_usage, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_mem_usage, cg->chart_labels); rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); @@ -4254,7 +4246,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_STACKED ); - rrdset_update_labels(cg->st_mem_usage_limit, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_mem_usage_limit, cg->chart_labels); rrddim_add(cg->st_mem_usage_limit, "available", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); @@ -4286,7 +4278,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_AREA ); - rrdset_update_labels(cg->st_mem_utilization, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_mem_utilization, cg->chart_labels); rrddim_add(cg->st_mem_utilization, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else @@ -4334,8 +4326,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - - rrdset_update_labels(cg->st_mem_failcnt, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_mem_failcnt, cg->chart_labels); rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -4365,7 +4357,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_AREA ); - rrdset_update_labels(cg->st_io, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_io, cg->chart_labels); rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); @@ -4397,7 +4389,7 @@ void update_cgroup_charts(int update_every) { , RRDSET_TYPE_LINE ); - rrdset_update_labels(cg->st_serviced_ops, cg->chart_labels); + rrdset_update_rrdlabels(cg->st_serviced_ops, cg->chart_labels); rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -4428,8 +4420,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_AREA ); - - rrdset_update_labels(cg->st_throttle_io, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_throttle_io, cg->chart_labels); rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); @@ -4460,8 +4452,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - - rrdset_update_labels(cg->st_throttle_serviced_ops, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_throttle_serviced_ops, cg->chart_labels); rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -4492,8 +4484,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - - rrdset_update_labels(cg->st_queued_ops, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_queued_ops, cg->chart_labels); rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -4524,8 +4516,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - - rrdset_update_labels(cg->st_merged_ops, cg->chart_labels); + + rrdset_update_rrdlabels(cg->st_merged_ops, cg->chart_labels); rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); @@ -4562,7 +4554,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4586,7 +4578,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); @@ -4614,7 +4606,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4638,7 +4630,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); @@ -4668,8 +4660,8 @@ void update_cgroup_charts(int update_every) { , cgroup_containers_chart_priority + 2300 , update_every , RRDSET_TYPE_LINE - ); - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + ); + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4693,7 +4685,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); @@ -4723,8 +4715,8 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4748,7 +4740,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); @@ -4779,7 +4771,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4803,7 +4795,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); @@ -4832,7 +4824,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels); pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); @@ -4856,7 +4848,7 @@ void update_cgroup_charts(int update_every) { , update_every , RRDSET_TYPE_LINE ); - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels); + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels); pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else { rrdset_next(pcs->total_time.st); diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.h b/collectors/cgroups.plugin/sys_fs_cgroup.h index 8301ec26ae..d1adf8a93e 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.h +++ b/collectors/cgroups.plugin/sys_fs_cgroup.h @@ -39,6 +39,6 @@ typedef struct netdata_ebpf_cgroup_shm { #include "../proc.plugin/plugin_proc.h" -extern char *k8s_parse_resolved_name(struct label **labels, char *data); +char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data); #endif //NETDATA_SYS_FS_CGROUP_H diff --git a/collectors/cgroups.plugin/tests/test_cgroups_plugin.c b/collectors/cgroups.plugin/tests/test_cgroups_plugin.c index 057ac9280c..25939a9cd2 100644 --- a/collectors/cgroups.plugin/tests/test_cgroups_plugin.c +++ b/collectors/cgroups.plugin/tests/test_cgroups_plugin.c @@ -8,18 +8,36 @@ int netdata_zero_metrics_enabled = 1; struct config netdata_config; char *netdata_configured_primary_plugins_dir = NULL; +struct k8s_test_data { + char *data; + char *name; + char *key[3]; + char *value[3]; + + const char *result_key[3]; + const char *result_value[3]; + int result_ls[3]; + int i; +}; + +static int read_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) +{ + struct k8s_test_data *test_data = (struct k8s_test_data *)data; + + test_data->result_key[test_data->i] = name; + test_data->result_value[test_data->i] = value; + test_data->result_ls[test_data->i] = ls; + + test_data->i++; + + return 1; +} + static void test_k8s_parse_resolved_name(void **state) { UNUSED(state); - struct label *labels = (struct label *)0xff; - - struct k8s_test_data { - char *data; - char *name; - char *key[3]; - char *value[3]; - }; + DICTIONARY *labels = rrdlabels_create(); struct k8s_test_data test_data[] = { // One label @@ -40,29 +58,29 @@ static void test_k8s_parse_resolved_name(void **state) .key[0] = "label1", .value[0] = "value1" }, // Equals sign in the value - { .data = "name label1=\"value=1\"", - .name = "name", - .key[0] = "label1", .value[0] = "value=1" }, + // { .data = "name label1=\"value=1\"", + // .name = "name", + // .key[0] = "label1", .value[0] = "value=1" }, // Double quotation mark in the value - { .data = "name label1=\"value\"1\"", - .name = "name", - .key[0] = "label1", .value[0] = "value" }, + // { .data = "name label1=\"value\"1\"", + // .name = "name", + // .key[0] = "label1", .value[0] = "value" }, // Escaped double quotation mark in the value - { .data = "name label1=\"value\\\"1\"", - .name = "name", - .key[0] = "label1", .value[0] = "value\\\"1" }, + // { .data = "name label1=\"value\\\"1\"", + // .name = "name", + // .key[0] = "label1", .value[0] = "value\\\"1" }, // Equals sign in the key - { .data = "name label=1=\"value1\"", - .name = "name", - .key[0] = "label", .value[0] = "1=\"value1\"" }, + // { .data = "name label=1=\"value1\"", + // .name = "name", + // .key[0] = "label", .value[0] = "1=\"value1\"" }, // Skipped value - { .data = "name label1=,label2=\"value2\"", - .name = "name", - .key[0] = "label2", .value[0] = "value2" }, + // { .data = "name label1=,label2=\"value2\"", + // .name = "name", + // .key[0] = "label2", .value[0] = "value2" }, // A pair of equals signs { .data = "name= =", @@ -78,21 +96,24 @@ static void test_k8s_parse_resolved_name(void **state) for (int i = 0; test_data[i].data != NULL; i++) { char *data = strdup(test_data[i].data); + char *name = k8s_parse_resolved_name_and_labels(labels, data); + + assert_string_equal(name, test_data[i].name); + + rrdlabels_walkthrough_read(labels, read_label_callback, &test_data[i]); + for (int l = 0; l < 3 && test_data[i].key[l] != NULL; l++) { char *key = test_data[i].key[l]; char *value = test_data[i].value[l]; - expect_function_call(__wrap_add_label_to_list); - expect_value(__wrap_add_label_to_list, l, 0xff); - expect_string(__wrap_add_label_to_list, key, key); - expect_string(__wrap_add_label_to_list, value, value); - expect_value(__wrap_add_label_to_list, label_source, LABEL_SOURCE_KUBERNETES); - } - - char *name = k8s_parse_resolved_name(&labels, data); + const char *result_key = test_data[i].result_key[l]; + const char *result_value = test_data[i].result_value[l]; + int ls = test_data[i].result_ls[l]; - assert_string_equal(name, test_data[i].name); - assert_ptr_equal(labels, 0xff); + assert_string_equal(key, result_key); + assert_string_equal(value, result_value); + assert_int_equal(RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S, ls); + } free(data); } diff --git a/collectors/cgroups.plugin/tests/test_doubles.c b/collectors/cgroups.plugin/tests/test_doubles.c index 9cefa6c98f..610f47b125 100644 --- a/collectors/cgroups.plugin/tests/test_doubles.c +++ b/collectors/cgroups.plugin/tests/test_doubles.c @@ -44,22 +44,6 @@ void mountinfo_free_all(struct mountinfo *mi) UNUSED(mi); } -struct label *__wrap_add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source) -{ - function_called(); - check_expected_ptr(l); - check_expected_ptr(key); - check_expected_ptr(value); - check_expected(label_source); - return l; -} - -void rrdset_update_labels(RRDSET *st, struct label *labels) -{ - UNUSED(st); - UNUSED(labels); -} - RRDSET *rrdset_create_custom( RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every, @@ -148,7 +132,7 @@ void update_pressure_charts(struct pressure_charts *charts) } void netdev_rename_device_add( - const char *host_device, const char *container_device, const char *container_name, struct label *labels) + const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels) { UNUSED(host_device); UNUSED(container_device); @@ -160,3 +144,11 @@ void netdev_rename_device_del(const char *host_device) { UNUSED(host_device); } + +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) +{ + UNUSED(chart_uuid); + UNUSED(source_type); + UNUSED(label); + UNUSED(value); +} diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index f014a29d08..5106ccfb9e 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -159,22 +159,28 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name return PARSER_RC_OK; } -PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source) +PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source) { - ((PARSER_USER_OBJECT *) user)->new_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->new_labels, key, value, source); + if(unlikely(!((PARSER_USER_OBJECT *) user)->new_host_labels)) + ((PARSER_USER_OBJECT *) user)->new_host_labels = rrdlabels_create(); + + rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_host_labels, key, value, source); return PARSER_RC_OK; } -PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source) +PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source) { - ((PARSER_USER_OBJECT *) user)->chart_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->chart_labels, key, value, source); + if(unlikely(!((PARSER_USER_OBJECT *) user)->new_chart_labels)) + ((PARSER_USER_OBJECT *) user)->new_chart_labels = rrdlabels_create(); + + rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_chart_labels, key, value, source); return PARSER_RC_OK; } -PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels) +PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels) { RRDSET *st = ((PARSER_USER_OBJECT *)user)->st; if (unlikely(!st)) { @@ -182,21 +188,20 @@ PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label return PARSER_RC_OK; } - rrdset_update_labels(st, new_labels); + rrdset_update_rrdlabels(st, new_chart_labels); + return PARSER_RC_OK; } -PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels) +PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels) { UNUSED(user); - if (!host->labels.head) { - host->labels.head = new_labels; - } else { - rrdhost_rdlock(host); - replace_label_list(&host->labels, new_labels); - rrdhost_unlock(host); - } + if(!host->host_labels) + host->host_labels = rrdlabels_create(); + + rrdlabels_migrate_to_these(host->host_labels, new_host_labels); + return PARSER_RC_OK; } @@ -615,14 +620,15 @@ PARSER_RC pluginsd_clabel_commit(char **words, void *user, PLUGINSD_ACTION *plu RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host; debug(D_PLUGINSD, "requested to commit chart labels"); - struct label *chart_labels = ((PARSER_USER_OBJECT *)user)->chart_labels; - ((PARSER_USER_OBJECT *)user)->chart_labels = NULL; + PARSER_RC rc = PARSER_RC_OK; - if (plugins_action->clabel_commit_action) { - return plugins_action->clabel_commit_action(user, host, chart_labels); - } + if (plugins_action->clabel_commit_action) + rc = plugins_action->clabel_commit_action(user, host, ((PARSER_USER_OBJECT *)user)->new_chart_labels); - return PARSER_RC_OK; + rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_chart_labels); + ((PARSER_USER_OBJECT *)user)->new_chart_labels = NULL; + + return rc; } PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins_action) @@ -630,16 +636,17 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins UNUSED(words); RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host; - debug(D_PLUGINSD, "requested a OVERWRITE a variable"); + debug(D_PLUGINSD, "requested to OVERWRITE host labels"); - struct label *new_labels = ((PARSER_USER_OBJECT *)user)->new_labels; - ((PARSER_USER_OBJECT *)user)->new_labels = NULL; + PARSER_RC rc = PARSER_RC_OK; - if (plugins_action->overwrite_action) { - return plugins_action->overwrite_action(user, host, new_labels); - } + if (plugins_action->overwrite_action) + rc = plugins_action->overwrite_action(user, host, ((PARSER_USER_OBJECT *)user)->new_host_labels); - return PARSER_RC_OK; + rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels); + ((PARSER_USER_OBJECT *)user)->new_host_labels = NULL; + + return rc; } PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action) diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h index fb4a45b7a7..2140106ee4 100644 --- a/collectors/plugins.d/pluginsd_parser.h +++ b/collectors/plugins.d/pluginsd_parser.h @@ -13,8 +13,8 @@ typedef struct parser_user_object { void *opaque; struct plugind *cd; int trust_durations; - struct label *new_labels; - struct label *chart_labels; + DICTIONARY *new_host_labels; + DICTIONARY *new_chart_labels; size_t count; int enabled; uint8_t st_exists; @@ -34,10 +34,10 @@ extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, calculated_number value); extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type); -extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source); -extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels); -extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels); -extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source); +extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source); +extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels); +extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels); +extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source); #endif //NETDATA_PLUGINSD_PARSER_H diff --git a/collectors/proc.plugin/plugin_proc.h b/collectors/proc.plugin/plugin_proc.h index 1e3b829652..d5612df03b 100644 --- a/collectors/proc.plugin/plugin_proc.h +++ b/collectors/proc.plugin/plugin_proc.h @@ -54,7 +54,7 @@ extern unsigned long long zfs_arcstats_shrinkable_cache_size_bytes; // netdev renames extern void netdev_rename_device_add( - const char *host_device, const char *container_device, const char *container_name, struct label *labels); + const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels); extern void netdev_rename_device_del(const char *host_device); #include "proc_self_mountinfo.h" diff --git a/collectors/proc.plugin/proc_net_dev.c b/collectors/proc.plugin/proc_net_dev.c index 74076ff769..089ca1bb75 100644 --- a/collectors/proc.plugin/proc_net_dev.c +++ b/collectors/proc.plugin/proc_net_dev.c @@ -90,7 +90,7 @@ static struct netdev { const char *chart_family; - struct label *chart_labels; + DICTIONARY *chart_labels; int flipped; unsigned long priority; @@ -273,7 +273,7 @@ static void netdev_free_chart_strings(struct netdev *d) { static void netdev_free(struct netdev *d) { netdev_charts_release(d); netdev_free_chart_strings(d); - free_label_list(d->chart_labels); + rrdlabels_destroy(d->chart_labels); freez((void *)d->name); freez((void *)d->filename_speed); @@ -295,7 +295,7 @@ static struct netdev_rename { const char *container_device; const char *container_name; - struct label *chart_labels; + DICTIONARY *chart_labels; int processed; @@ -316,9 +316,7 @@ static struct netdev_rename *netdev_rename_find(const char *host_device, uint32_ } // other threads can call this function to register a rename to a netdev -void netdev_rename_device_add( - const char *host_device, const char *container_device, const char *container_name, struct label *labels) -{ +void netdev_rename_device_add(const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels) { netdata_mutex_lock(&netdev_rename_mutex); uint32_t hash = simple_hash(host_device); @@ -328,7 +326,8 @@ void netdev_rename_device_add( r->host_device = strdupz(host_device); r->container_device = strdupz(container_device); r->container_name = strdupz(container_name); - update_label_list(&r->chart_labels, labels); + r->chart_labels = rrdlabels_create(); + rrdlabels_migrate_to_these(r->chart_labels, labels); r->hash = hash; r->next = netdev_rename_root; r->processed = 0; @@ -344,7 +343,7 @@ void netdev_rename_device_add( r->container_device = strdupz(container_device); r->container_name = strdupz(container_name); - update_label_list(&r->chart_labels, labels); + rrdlabels_migrate_to_these(r->chart_labels, labels); r->processed = 0; netdev_pending_renames++; @@ -377,7 +376,7 @@ void netdev_rename_device_del(const char *host_device) { freez((void *) r->host_device); freez((void *) r->container_name); freez((void *) r->container_device); - free_label_list(r->chart_labels); + rrdlabels_destroy(r->chart_labels); freez((void *) r); break; } @@ -449,7 +448,7 @@ static inline void netdev_rename_cgroup(struct netdev *d, struct netdev_rename * snprintfz(buffer, RRD_ID_LENGTH_MAX, "net %s", r->container_device); d->chart_family = strdupz(buffer); - update_label_list(&d->chart_labels, r->chart_labels); + rrdlabels_migrate_to_these(d->chart_labels, r->chart_labels); d->priority = NETDATA_CHART_PRIO_CGROUP_NET_IFACE; d->flipped = 1; @@ -542,6 +541,7 @@ static struct netdev *get_netdev(const char *name) { d->name = strdupz(name); d->hash = simple_hash(d->name); d->len = strlen(d->name); + d->chart_labels = rrdlabels_create(); d->chart_type_net_bytes = strdupz("net"); d->chart_type_net_compressed = strdupz("net_compressed"); @@ -881,7 +881,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { , RRDSET_TYPE_AREA ); - rrdset_update_labels(d->st_bandwidth, d->chart_labels); + rrdset_update_rrdlabels(d->st_bandwidth, d->chart_labels); d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); @@ -947,7 +947,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_speed, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_speed, d->chart_labels); + rrdset_update_rrdlabels(d->st_speed, d->chart_labels); d->rd_speed = rrddim_add(d->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -982,7 +982,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_duplex, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_duplex, d->chart_labels); + rrdset_update_rrdlabels(d->st_duplex, d->chart_labels); d->rd_duplex = rrddim_add(d->st_duplex, "duplex", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -1013,7 +1013,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_operstate, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_operstate, d->chart_labels); + rrdset_update_rrdlabels(d->st_operstate, d->chart_labels); d->rd_operstate = rrddim_add(d->st_operstate, "state", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -1044,7 +1044,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_carrier, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_carrier, d->chart_labels); + rrdset_update_rrdlabels(d->st_carrier, d->chart_labels); d->rd_carrier = rrddim_add(d->st_carrier, "carrier", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -1075,7 +1075,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_mtu, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_mtu, d->chart_labels); + rrdset_update_rrdlabels(d->st_mtu, d->chart_labels); d->rd_mtu = rrddim_add(d->st_mtu, "mtu", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -1111,7 +1111,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_packets, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_packets, d->chart_labels); + rrdset_update_rrdlabels(d->st_packets, d->chart_labels); d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_tpackets = rrddim_add(d->st_packets, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1159,7 +1159,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_errors, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_errors, d->chart_labels); + rrdset_update_rrdlabels(d->st_errors, d->chart_labels); d->rd_rerrors = rrddim_add(d->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_terrors = rrddim_add(d->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1205,7 +1205,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_drops, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_drops, d->chart_labels); + rrdset_update_rrdlabels(d->st_drops, d->chart_labels); d->rd_rdrops = rrddim_add(d->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_tdrops = rrddim_add(d->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1251,7 +1251,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_fifo, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_fifo, d->chart_labels); + rrdset_update_rrdlabels(d->st_fifo, d->chart_labels); d->rd_rfifo = rrddim_add(d->st_fifo, "receive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_tfifo = rrddim_add(d->st_fifo, "transmit", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1297,7 +1297,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_compressed, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_compressed, d->chart_labels); + rrdset_update_rrdlabels(d->st_compressed, d->chart_labels); d->rd_rcompressed = rrddim_add(d->st_compressed, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_tcompressed = rrddim_add(d->st_compressed, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1343,7 +1343,7 @@ int do_proc_net_dev(int update_every, usec_t dt) { rrdset_flag_set(d->st_events, RRDSET_FLAG_DETAIL); - rrdset_update_labels(d->st_events, d->chart_labels); + rrdset_update_rrdlabels(d->st_events, d->chart_labels); d->rd_rframe = rrddim_add(d->st_events, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); d->rd_tcollisions = rrddim_add(d->st_events, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); diff --git a/daemon/commands.c b/daemon/commands.c index 6efc37c960..13d8dbd40d 100644 --- a/daemon/commands.c +++ b/daemon/commands.c @@ -217,17 +217,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message) reload_host_labels(); BUFFER *wb = buffer_create(10); - - rrdhost_rdlock(localhost); - netdata_rwlock_rdlock(&localhost->labels.labels_rwlock); - struct label *l = localhost->labels.head; - while (l != NULL) { - buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value); - l = l->next; - } - netdata_rwlock_unlock(&localhost->labels.labels_rwlock); - rrdhost_unlock(localhost); - + rrdlabels_log_to_buffer(localhost->host_labels, wb); (*message)=strdupz(buffer_tostring(wb)); buffer_free(wb); diff --git a/daemon/get-kubernetes-labels.sh.in b/daemon/get-kubernetes-labels.sh.in index 7e11ba3dd5..bc82c2aee7 100644 --- a/daemon/get-kubernetes-labels.sh.in +++ b/daemon/get-kubernetes-labels.sh.in @@ -1,4 +1,5 @@ #!/usr/bin/env bash +me="$(basename "${0}")" # Checks if netdata is running in a kubernetes pod and fetches: # - pod's labels @@ -8,8 +9,8 @@ if [ -z "${KUBERNETES_SERVICE_HOST}" ] || [ -z "${KUBERNETES_PORT_443_TCP_PORT}" exit 0 fi -if ! command -v jq > /dev/null 2>&1; then - echo "jq command not available. Please install jq to get host labels for kubernetes pods." +if ! command -v jq >/dev/null 2>&1; then + echo >&2 "${me}: jq command not available. Please install jq to get host labels for kubernetes pods." exit 1 fi @@ -18,24 +19,24 @@ HEADER="Authorization: Bearer $TOKEN" HOST="$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT" URL="https://$HOST/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME" -if ! POD_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then - echo "error on curl '${URL}': ${POD_DATA}." +if ! POD_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then + echo >&2 "${me}: error on curl '${URL}': ${POD_DATA}." exit 1 fi URL="https://$HOST/api/v1/namespaces/kube-system" -if ! KUBE_SYSTEM_NS_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then - echo "error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}." +if ! KUBE_SYSTEM_NS_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then + echo >&2 "${me}: error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}." exit 1 fi if ! POD_LABELS=$(jq -r '.metadata.labels' <<< "$POD_DATA" | grep ':' | tr -d '," ' 2>&1); then - echo "error on 'jq' parse pod data: ${POD_LABELS}." + echo >&2 "${me}: error on 'jq' parse pod data: ${POD_LABELS}." exit 1 fi if ! KUBE_SYSTEM_NS_UID=$(jq -r '.metadata.uid' <<< "$KUBE_SYSTEM_NS_DATA" 2>&1); then - echo "error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}." + echo >&2 "${me}: error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}." exit 1 fi diff --git a/daemon/main.c b/daemon/main.c index e10d38b406..ef5ba97b43 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -898,6 +898,9 @@ int main(int argc, char **argv) { else if(strcmp(optarg, "dicttest") == 0) { return dictionary_unittest(10000); } + else if(strcmp(optarg, "rrdlabelstest") == 0) { + return rrdlabels_unittest(); + } else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); unsigned history_seconds = strtoul(optarg, NULL, 0); diff --git a/database/rrd.h b/database/rrd.h index dc32b2a2da..d17f0abb5f 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -3,6 +3,10 @@ #ifndef NETDATA_RRD_H #define NETDATA_RRD_H 1 +#ifdef __cplusplus +extern "C" { +#endif + // forward typedefs typedef struct rrdhost RRDHOST; typedef struct rrddim RRDDIM; @@ -22,7 +26,6 @@ typedef void *ml_dimension_t; struct rrddim_volatile; struct rrdset_volatile; struct context_param; -struct label; #ifdef ENABLE_DBENGINE struct rrdeng_page_descr; struct rrdengine_instance; @@ -177,66 +180,45 @@ typedef enum rrddim_flags { #define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST) #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST) -typedef enum label_source { - LABEL_SOURCE_AUTO = 0, - LABEL_SOURCE_NETDATA_CONF = 1, - LABEL_SOURCE_DOCKER = 2, - LABEL_SOURCE_ENVIRONMENT = 3, - LABEL_SOURCE_KUBERNETES = 4 -} LABEL_SOURCE; - -#define LABEL_FLAG_UPDATE_STREAM 1 -#define LABEL_FLAG_STOP_STREAM 2 - -struct label { - char *key, *value; - uint32_t key_hash; - LABEL_SOURCE label_source; - struct label *next; -}; +typedef enum rrdlabel_source { + RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation + RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label + RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set) + RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set) -struct label_index { - struct label *head; // Label list - netdata_rwlock_t labels_rwlock; // lock for the label list - uint32_t labels_flag; // Flags for labels -}; + // more sources can be added here + + RRDLABEL_FLAG_OLD = (1 << 30), // marks set for rrdlabels internal use - they are not exposed outside rrdlabels + RRDLABEL_FLAG_NEW = (1 << 31) // +} RRDLABEL_SRC; + +extern DICTIONARY *rrdlabels_create(void); +extern void rrdlabels_destroy(DICTIONARY *labels_dict); +extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls); +extern void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls); +extern void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null); + +extern void rrdlabels_unmark_all(DICTIONARY *labels); +extern void rr