Labels with dictionary (#13070)

* squashed and rebased to master

* fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h

* added unittest for UTF-8 multibyte sanitization

* Fix unit test compilation

* Fix CMake build

* remove double sanitizer for opentsdb; cleanup sanitize_json_string()

* rename error_description to error_message to avoid conflict with json-c

* revert last and undef error_description from json-c

* more unittests; attempt to fix protobuf map issue

* get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer

* added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted

* better sorted dictionary checking

* proper unittesting for sorted dictionaries

* call dictionary deletion callback when destroying the dictionary

* remove obsolete variable

* Fix exporting unit tests

* Fix k8s label parsing test

* workaround for cmocka and strdupz()

* Bypass cmocka memory allocation check

* Revert "Bypass cmocka memory allocation check"

This reverts commit 4c49923839d9229bea23ca914dd8a0be1ebe2bf4.

* Revert "workaround for cmocka and strdupz()"

This reverts commit 7bebee04801db1865c748a7896d5fa54bb7104a5.

* Bypass cmocka memory allocation checks

* respect json formatting for chart labels

* cloud sends colons

* print the value only once

* allow parenthesis in values and spaces; make stream sender send quotes for values

Co-authored-by: Vladimir Kobal <vlad@prokk.net>

3 files changed, 35 insertions, 58 deletions

diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 7097a5b778..af097a6712 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -19,6 +19,21 @@ static int value_list_output(const char *name, void *entry, void *data) {
     return 0;
 }
 
+static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    (void)ls;
+    DICTIONARY *dict = (DICTIONARY *)data;
+    char n[RRD_ID_LENGTH_MAX * 2 + 2];
+    char output[RRD_ID_LENGTH_MAX * 2 + 8];
+    char v[RRD_ID_LENGTH_MAX * 2 + 1];
+
+    sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2);
+    int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v);
+    snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v);
+    dictionary_set(dict, n, output, len + 1);
+
+    return 1;
+}
+
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
     QUERY_PARAMS *rrdset_query_data)
 {
@@ -122,7 +137,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         char name[RRD_ID_LENGTH_MAX * 2 + 2];
         char output[RRD_ID_LENGTH_MAX * 2 + 8];
-        char value[RRD_ID_LENGTH_MAX * 2 + 1];
 
         struct value_output co = {.c = 0, .wb = wb};
 
@@ -153,19 +167,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
         dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
         for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
             st = rd->rrdset;
-            if (likely(st->state)) {
-                struct label_index *labels = &st->state->labels;
-                if (labels->head) {
-                    netdata_rwlock_rdlock(&labels->labels_rwlock);
-                    for (struct label *label = labels->head; label; label = label->next) {
-                        sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
-                        int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
-                        snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
-                        dictionary_set(dict, name, output, len + 1);
-                    }
-                    netdata_rwlock_unlock(&labels->labels_rwlock);
-                }
-            }
+            if (st->state && st->state->chart_labels)
+                rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict);
         }
         dictionary_walkthrough_read(dict, value_list_output, &co);
         dictionary_destroy(dict);
@@ -207,8 +210,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             char *label_key = NULL;
             int keys = 0;
             while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
-                uint32_t key_hash = simple_hash(label_key);
-                struct label *current_label;
 
                 if (keys)
                     buffer_strcat(wb, ", ");
@@ -223,13 +224,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                     if (i)
                         buffer_strcat(wb, ", ");
 
-                    current_label = rrdset_lookup_label_key(rd->rrdset, label_key, key_hash);
-                    if (current_label) {
-                        buffer_strcat(wb, sq);
-                        buffer_strcat(wb, current_label->value);
-                        buffer_strcat(wb, sq);
-                    } else
-                        buffer_strcat(wb, "null");
+                    rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null");
                     i++;
                 }
                 if (!i) {
diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c
index c83b22e639..26b9c1d481 100644
--- a/web/api/formatters/rrdset2json.c
+++ b/web/api/formatters/rrdset2json.c
@@ -4,8 +4,10 @@
 
 void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
 {
+    if(unlikely(!st->state || !st->state->chart_labels))
+        return;
+
     char tabs[11];
-    struct label_index *labels = &st->state->labels;
 
     if (indentation > 10)
         indentation = 10;
@@ -16,20 +18,8 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
         indentation--;
     }
 
-    int count = 0;
-    netdata_rwlock_rdlock(&labels->labels_rwlock);
-    for (struct label *label = labels->head; label; label = label->next) {
-        if(count > 0) buffer_strcat(wb, ",\n");
-        buffer_strcat(wb, tabs);
-
-        char value[CONFIG_MAX_VALUE * 2 + 1];
-        sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
-        buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
-
-        count++;
-    }
+    rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
     buffer_strcat(wb, "\n");
-    netdata_rwlock_unlock(&labels->labels_rwlock);
 }
 
 // generate JSON for the /api/v1/chart API call
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index cb73f7c022..ceb52dece3 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -392,6 +392,7 @@ void fix_google_param(char *s) {
     }
 }
 
+
 // returns the HTTP code
 inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url) {
     debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
@@ -528,18 +529,23 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
 
         uint32_t context_hash = simple_hash(context);
 
+        SIMPLE_PATTERN *chart_label_key_pattern = NULL;
+        if(chart_label_key)
+            chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
+
+        SIMPLE_PATTERN *chart_labels_filter_pattern = NULL;
+        if(chart_labels_filter)
+            chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
+
         rrdhost_rdlock(host);
-        char *words[MAX_CHART_LABELS_FILTER];
-        uint32_t hash_key_list[MAX_CHART_LABELS_FILTER];
-        int word_count = 0;
         rrdset_foreach_read(st1, host) {
             if (st1->hash_context == context_hash && !strcmp(st1->context, context) &&
-                (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) &&
-                (!chart_labels_filter ||
-                 rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER)))
+                (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) &&
+                (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':')))
                     build_context_param_list(owa, &context_param_list, st1);
         }
         rrdhost_unlock(host);
+
         if (likely(context_param_list && context_param_list->rd))  // Just set the first one
             st = context_param_list->rd->rrdset;
         else {
@@ -964,22 +970,8 @@ inline void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation) {
         indentation--;
     }
 
-    int count = 0;
-    rrdhost_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if(count > 0) buffer_strcat(wb, ",\n");
-        buffer_strcat(wb, tabs);
-
-        char value[CONFIG_MAX_VALUE * 2 + 1];
-        sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
-        buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
-
-        count++;
-    }
+    rrdlabels_to_buffer(host->host_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
     buffer_strcat(wb, "\n");
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
-    rrdhost_unlock(host);
 }
 
 extern int aclk_connected;