Labels with dictionary (#13070)

* squashed and rebased to master

* fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h

* added unittest for UTF-8 multibyte sanitization

* Fix unit test compilation

* Fix CMake build

* remove double sanitizer for opentsdb; cleanup sanitize_json_string()

* rename error_description to error_message to avoid conflict with json-c

* revert last and undef error_description from json-c

* more unittests; attempt to fix protobuf map issue

* get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer

* added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted

* better sorted dictionary checking

* proper unittesting for sorted dictionaries

* call dictionary deletion callback when destroying the dictionary

* remove obsolete variable

* Fix exporting unit tests

* Fix k8s label parsing test

* workaround for cmocka and strdupz()

* Bypass cmocka memory allocation check

* Revert "Bypass cmocka memory allocation check"

This reverts commit 4c49923839d9229bea23ca914dd8a0be1ebe2bf4.

* Revert "workaround for cmocka and strdupz()"

This reverts commit 7bebee04801db1865c748a7896d5fa54bb7104a5.

* Bypass cmocka memory allocation checks

* respect json formatting for chart labels

* cloud sends colons

* print the value only once

* allow parenthesis in values and spaces; make stream sender send quotes for values

Co-authored-by: Vladimir Kobal <vlad@prokk.net>

15 files changed, 160 insertions, 180 deletions

diff --git a/exporting/check_filters.c b/exporting/check_filters.c
index f7eba22db3..726fd02a1a 100644
--- a/exporting/check_filters.c
+++ b/exporting/check_filters.c
@@ -2,6 +2,14 @@
 
 #include "exporting_engine.h"
 
+
+bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    (void)name;
+    (void)value;
+    struct instance *instance = (struct instance *)data;
+    return should_send_label(instance, ls);
+}
+
 /**
  * Check if the connector instance should export the host metrics
  *
diff --git a/exporting/clean_connectors.c b/exporting/clean_connectors.c
index 4af1219a63..e935637417 100644
--- a/exporting/clean_connectors.c
+++ b/exporting/clean_connectors.c
@@ -33,7 +33,7 @@ static void clean_instance_config(struct instance_config *config)
 void clean_instance(struct instance *instance)
 {
     clean_instance_config(&instance->config);
-    buffer_free(instance->labels);
+    buffer_free(instance->labels_buffer);
 
     uv_cond_destroy(&instance->cond_var);
     // uv_mutex_destroy(&instance->mutex);
diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 20f260c15b..0feb38d2b3 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -4,7 +4,6 @@
 #define NETDATA_EXPORTING_ENGINE_H 1
 
 #include "daemon/common.h"
-
 #include <uv.h>
 
 #define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value)
@@ -40,11 +39,11 @@ extern const char *global_exporting_prefix;
 #define sending_labels_configured(instance)                                                                            \
     (instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))
 
-#define should_send_label(instance, label)                                                                             \
+#define should_send_label(instance, label_source)                                                                      \
     ((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS &&                                            \
-      label->label_source == LABEL_SOURCE_NETDATA_CONF) ||                                                             \
+      label_source & RRDLABEL_SRC_CONFIG) ||                                                                           \
      (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS &&                                             \
-      label->label_source != LABEL_SOURCE_NETDATA_CONF))
+      label_source & RRDLABEL_SRC_AUTO))
 
 typedef enum exporting_connector_types {
     EXPORTING_CONNECTOR_TYPE_UNKNOWN,                 // Invalid type
@@ -205,7 +204,7 @@ struct instance {
     int skip_host;
     int skip_chart;
 
-    BUFFER *labels;
+    BUFFER *labels_buffer;
 
     time_t after;
     time_t before;
diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 84d4febf13..9dbaf21dd4 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -71,7 +71,7 @@ int init_graphite_instance(struct instance *instance)
  * @param len the maximum number of characters copied.
  */
 
-void sanitize_graphite_label_value(char *dst, char *src, size_t len)
+void sanitize_graphite_label_value(char *dst, const char *src, size_t len)
 {
     while (*src != '\0' && len) {
         if (isspace(*src) || *src == ';' || *src == '~')
@@ -91,29 +91,19 @@ void sanitize_graphite_label_value(char *dst, char *src, size_t len)
  * @param host a data collecting host.
  * @return Always returns 0.
  */
+
 int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
 {
-    if (!instance->labels)
-        instance->labels = buffer_create(1024);
+    if (!instance->labels_buffer)
+        instance->labels_buffer = buffer_create(1024);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
 
-    rrdhost_check_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if (!should_send_label(instance, label))
-            continue;
-
-        char value[CONFIG_MAX_VALUE + 1];
-        sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE);
-
-        if (*value) {
-            buffer_strcat(instance->labels, ";");
-            buffer_sprintf(instance->labels, "%s=%s", label->key, value);
-        }
-    }
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
+    buffer_strcat(instance->labels_buffer, ";");
+    rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", ";",
+                        exporting_labels_filter_callback, instance,
+                        NULL, sanitize_graphite_label_value);
 
     return 0;
 }
@@ -151,7 +141,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
         dimension_name,
         (host->tags) ? ";" : "",
         (host->tags) ? host->tags : "",
-        (instance->labels) ? buffer_tostring(instance->labels) : "",
+        (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
         rd->last_collected_value,
         (unsigned long long)rd->last_collected_time.tv_sec);
 
@@ -197,7 +187,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
         dimension_name,
         (host->tags) ? ";" : "",
         (host->tags) ? host->tags : "",
-        (instance->labels) ? buffer_tostring(instance->labels) : "",
+        (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
         value,
         (unsigned long long)last_t);
 
diff --git a/exporting/graphite/graphite.h b/exporting/graphite/graphite.h
index 993c12e57a..79f87e46ed 100644
--- a/exporting/graphite/graphite.h
+++ b/exporting/graphite/graphite.h
@@ -7,7 +7,7 @@
 
 int init_graphite_instance(struct instance *instance);
 
-void sanitize_graphite_label_value(char *dst, char *src, size_t len);
+void sanitize_graphite_label_value(char *dst, const char *src, size_t len);
 int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host);
 
 int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
diff --git a/exporting/json/json.c b/exporting/json/json.c
index 50278c5b82..c8db216e4a 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -113,34 +113,20 @@ int init_json_http_instance(struct instance *instance)
  * @param host a data collecting host.
  * @return Always returns 0.
  */
+
 int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
 {
-    if (!instance->labels)
-        instance->labels = buffer_create(1024);
+    if (!instance->labels_buffer)
+        instance->labels_buffer = buffer_create(1024);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
 
-    buffer_strcat(instance->labels, "\"labels\":{");
-
-    int count = 0;
-    rrdhost_check_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if (!should_send_label(instance, label))
-            continue;
-
-        char value[CONFIG_MAX_VALUE * 2 + 1];
-        sanitize_json_string(value, label->value, CONFIG_MAX_VALUE);
-        if (count > 0)
-            buffer_strcat(instance->labels, ",");
-        buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
-
-        count++;
-    }
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
-    buffer_strcat(instance->labels, "},");
+    buffer_strcat(instance->labels_buffer, "\"labels\":{");
+    rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
+                        exporting_labels_filter_callback, instance,
+                        NULL, sanitize_json_string);
+    buffer_strcat(instance->labels_buffer, "},");
 
     return 0;
 }
@@ -203,7 +189,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
         tags_pre,
         tags,
         tags_post,
-        instance->labels ? buffer_tostring(instance->labels) : "",
+        instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
 
         st->id,
         st->name,
@@ -288,7 +274,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
         tags_pre,
         tags,
         tags_post,
-        instance->labels ? buffer_tostring(instance->labels) : "",
+        instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
 
         st->id,
         st->name,
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index 7ed88fd6d8..d6ce98ce45 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -124,13 +124,14 @@ int init_opentsdb_http_instance(struct instance *instance)
  * @param len the maximum number of characters copied.
  */
 
-void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
+void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len)
 {
     while (*src != '\0' && len) {
-        if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
+        if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
             *dst++ = *src;
         else
             *dst++ = '_';
+
         src++;
         len--;
     }
@@ -144,28 +145,18 @@ void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
  * @param host a data collecting host.
  * @return Always returns 0.
  */
-int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host)
-{
-    if (!instance->labels)
-        instance->labels = buffer_create(1024);
+
+int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) {
+    if(!instance->labels_buffer)
+        instance->labels_buffer = buffer_create(1024);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
 
-    rrdhost_check_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if (!should_send_label(instance, label))
-            continue;
-
-        char value[CONFIG_MAX_VALUE + 1];
-        sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE);
-
-        if (*value)
-            buffer_sprintf(instance->labels, " %s=%s", label->key, value);
-    }
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
+    buffer_strcat(instance->labels_buffer, " ");
+    rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ",
+                        exporting_labels_filter_callback, instance,
+                        NULL, sanitize_opentsdb_label_value);
     return 0;
 }
 
@@ -204,7 +195,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
         (host == localhost) ? instance->config.hostname : host->hostname,
         (host->tags) ? " " : "",
         (host->tags) ? host->tags : "",
-        (instance->labels) ? buffer_tostring(instance->labels) : "");
+        (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
 
     return 0;
 }
@@ -250,7 +241,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
         (host == localhost) ? instance->config.hostname : host->hostname,
         (host->tags) ? " " : "",
         (host->tags) ? host->tags : "",
-        (instance->labels) ? buffer_tostring(instance->labels) : "");
+        (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
 
     return 0;
 }
@@ -287,33 +278,18 @@ void opentsdb_http_prepare_header(struct instance *instance)
  * @param host a data collecting host.
  * @return Always returns 0.
  */
-int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host)
-{
-    if (!instance->labels)
-        instance->labels = buffer_create(1024);
+
+int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) {
+    if (!instance->labels_buffer)
+        instance->labels_buffer = buffer_create(1024);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
 
-    rrdhost_check_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if (!should_send_label(instance, label))
-            continue;
-
-        char escaped_value[CONFIG_MAX_VALUE * 2 + 1];
-        sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE);
-
-        char value[CONFIG_MAX_VALUE + 1];
-        sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE);
-
-        if (*value) {
-            buffer_strcat(instance->labels, ",");
-            buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
-        }
-    }
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
+    buffer_strcat(instance->labels_buffer, ",");
+    rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
+                        exporting_labels_filter_callback, instance,
+                        NULL, sanitize_opentsdb_label_value);
     return 0;
 }
 
@@ -362,7 +338,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
         (host == localhost) ? instance->config.hostname : host->hostname,
         (host->tags) ? " " : "",
         (host->tags) ? host->tags : "",
-        instance->labels ? buffer_tostring(instance->labels) : "");
+        instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
 
     return 0;
 }
@@ -418,7 +394,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
         (host == localhost) ? instance->config.hostname : host->hostname,
         (host->tags) ? " " : "",
         (host->tags) ? host->tags : "",
-        instance->labels ? buffer_tostring(instance->labels) : "");
+        instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
 
     return 0;
 }
diff --git a/exporting/opentsdb/opentsdb.h b/exporting/opentsdb/opentsdb.h
index d53a5054f5..b544ba8c16 100644
--- a/exporting/opentsdb/opentsdb.h
+++ b/exporting/opentsdb/opentsdb.h
@@ -8,7 +8,7 @@
 int init_opentsdb_telnet_instance(struct instance *instance);
 int init_opentsdb_http_instance(struct instance *instance);
 
-void sanitize_opentsdb_label_value(char *dst, char *src, size_t len);
+void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len);
 int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host);
 int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host);
 
diff --git a/exporting/process_data.c b/exporting/process_data.c
index c77b7ad4a5..548a7c8bf7 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -358,8 +358,8 @@ int flush_host_labels(struct instance *instance, RRDHOST *host)
 {
     (void)host;
 
-    if (instance->labels)
-        buffer_flush(instance->labels);
+    if (instance->labels_buffer)
+        buffer_flush(instance->labels_buffer);
 
     return 0;
 }
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index c7f3f1d389..95795742b8 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -290,35 +290,44 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
  * @param instance an instance data structure.
  * @param host a data collecting host.
  */
-void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
-{
-    if (unlikely(!sending_labels_configured(instance)))
-        return;
 
-    if (!instance->labels)
-        instance->labels = buffer_create(1024);
+struct format_prometheus_label_callback {
+    struct instance *instance;
+    size_t count;
+};
 
-    int count = 0;
-    rrdhost_check_rdlock(host);
-    netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-    for (struct label *label = host->labels.head; label; label = label->next) {
-        if (!should_send_label(instance, label))
-            continue;
+static int format_prometheus_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    struct format_prometheus_label_callback *d = (struct format_prometheus_label_callback *)data;
 
-        char key[PROMETHEUS_ELEMENT_MAX + 1];
-        char value[PROMETHEUS_ELEMENT_MAX + 1];
+    if (!should_send_label(d->instance, ls)) return 0;
 
-        prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
-        prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
+    char k[PROMETHEUS_ELEMENT_MAX + 1];
+    char v[PROMETHEUS_ELEMENT_MAX + 1];
 
-        if (*key && *value) {
-            if (count > 0)
-                buffer_strcat(instance->labels, ",");
-            buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
-            count++;
-        }
+    prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
+    prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
+
+    if (*k && *v) {
+        if (d->count > 0) buffer_strcat(d->instance->labels_buffer, ",");
+        buffer_sprintf(d->instance->labels_buffer, "%s=\"%s\"", k, v);
+        d->count++;
     }
-    netdata_rwlock_unlock(&host->labels.labels_rwlock);
+    return 1;
+}
+
+void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
+{
+    if (unlikely(!sending_labels_configured(instance)))
+        return;
+
+    if (!instance->labels_buffer)
+        instance->labels_buffer = buffer_create(1024);
+
+    struct format_prometheus_label_callback tmp = {
+        .instance = instance,
+        .count = 0
+    };
+    rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp);
 }
 
 struct host_variables_callback_options {
@@ -534,13 +543,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
 
     char labels[PROMETHEUS_LABELS_MAX + 1] = "";
     if (allhosts) {
-        if (instance->labels && buffer_tostring(instance->labels)) {
+        if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(
                     wb,
                     "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
                     hostname,
-                    buffer_tostring(instance->labels),
+                    buffer_tostring(instance->labels_buffer),
                     now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
@@ -548,45 +557,45 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                     wb,
                     "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
                     hostname,
-                    buffer_tostring(instance->labels),
+                    buffer_tostring(instance->labels_buffer),
                     now_realtime_usec() / USEC_PER_MS);
             } else {
                 buffer_sprintf(
-                    wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
+                    wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
 
                 // deprecated, exists only for compatibility with older queries
                 buffer_sprintf(
-                    wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
+                    wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
             }
         }
 
         snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
     } else {
-        if (instance->labels && buffer_tostring(instance->labels)) {
+        if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(
                     wb,
                     "netdata_host_tags_info{%s} 1 %llu\n",
-                    buffer_tostring(instance->labels),
+                    buffer_tostring(instance->labels_buffer),
                     now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
                 buffer_sprintf(
                     wb,
                     "netdata_host_tags{%s} 1 %llu\n",
-                    buffer_tostring(instance->labels),
+                    buffer_tostring(instance->labels_buffer),
                     now_realtime_usec() / USEC_PER_MS);
             } else {
-                buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
+                buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels_buffer));
 
                 // deprecated, exists only for compatibility with older queries
-                buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
+                buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels_buffer));
             }
         }
     }
 
-    if (instance->labels)
-        buffer_flush(instance->labels);
+    if (instance->labels_buffer)
+        buffer_flush(instance->labels_buffer);
 
     // send custom variables set for the host
     if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index 59a488e1b5..ac00d3b50e 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -141,6 +141,26 @@ int init_prometheus_remote_write_instance(struct instance *instance)
  * @param host a data collecting host.
  * @return Always returns 0.
  */
+
+struct format_remote_write_label_callback {
+    struct instance *instance;
+    void *write_request;
+};
+
+static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+    struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data;
+
+    if (!should_send_label(d->instance, ls)) return 0;
+
+    char k[PROMETHEUS_ELEMENT_MAX + 1];
+    char v[PROMETHEUS_ELEMENT_MAX + 1];
+
+    prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
+    prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
+    add_label(d->write_request, k, v);
+    return 1;
+}
+
 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
 {
     struct simple_connector_data *simple_connector_data =
@@ -159,21 +179,11 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
         "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
 
     if (unlikely(sending_labels_configured(instance))) {
-        rrdhost_check_rdlock(host);
-        netdata_rwlock_rdlock(&host->labels.labels_rwlock);
-        for (struct label *label = host->labels.head; label; label = label->next) {
-            if (!should_send_label(instance, label))
-                continue;
-
-            char key[PROMETHEUS_ELEMENT_MAX + 1];
-            prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
-
-            char value[PROMETHEUS_ELEMENT_MAX + 1];
-            prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
-
-            add_label(connector_specific_data->write_request, key, value);
-        }
-        netdata_rwlock_unlock(&host->labels.labels_rwlock);
+        struct format_remote_write_label_callback tmp = {
+            .write_request = connector_specific_data->write_request,
+            .instance = instance
+        };
+        rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp);
     }
 
     return 0;
diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c
index 501fc405c9..4f5bd4b277 100644
--- a/exporting/tests/exporting_fixtures.c
+++ b/exporting/tests/exporting_fixtures.c
@@ -41,17 +41,9 @@ int setup_rrdhost()
 
     localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
 
-    struct label *label = calloc(1, sizeof(struct label));
-    label->key = strdupz("key1");
-    label->value = strdupz("value1");
-    label->label_source = LABEL_SOURCE_NETDATA_CONF;
-    localhost->labels.head = label;
-
-    label = calloc(1, sizeof(struct label));
-    label->key = strdupz("key2");
-    label->value = strdupz("value2");
-    label->label_source = LABEL_SOURCE_AUTO;
-    localhost->labels.head->next = label;
+    localhost->host_labels = rrdlabels_create();
+    rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG);
+    rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG);
 
     localhost->rrdset_root = calloc(1, sizeof(RRDSET));
     RRDSET *st = localhost->rrdset_root;
@@ -94,12 +86,7 @@ int teardown_rrdhost()
     free((void *)st->name);
     free(st);
 
-    free(localhost->labels.head->next->key);
-    free(localhost->labels.head->next->value);
-    free(localhost->labels.head->next);
-    free(localhost->labels.head->key);
-    free(localhost->labels.head->value);
-    free(localhost->labels.head);
+    rrdlabels_destroy(localhost->host_labels);
 
     free((void *)localhost->tags);
     free(localhost);
@@ -124,7 +111,7 @@ int teardown_initialized_engine(void **state)
     struct engine *engine = *state;
 
     teardown_rrdhost();
-    buffer_free(engine->instance_root->labels);
+    buffer_free(engine->instance_root->labels_buffer);
     buffer_free(engine->instance_root->buffer);
     teardown_configured_engine(state);
 
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index a9a1843367..8fba62e783 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -245,3 +245,11 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
 
     function_called();
 }
+
+void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
+{
+    (void)chart_uuid;
+    (void)source_type;
+    (void)label;
+    (void)value;
+}
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index 6bb7d2efde..d0b846429d 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -381,7 +381,7 @@ static void test_prepare_buffers(void **state)
     expect_value(__mock_end_batch_formatting, instance, instance);
     will_return(__mock_end_batch_formatting, 0);
 
-    assert_int_equal(__real_prepare_buffers(engine), 0);
+    __real_prepare_buffers(engine);
 
     assert_int_equal(instance->stats.buffered_metrics, 1);
 
@@ -393,7 +393,7 @@ static void test_prepare_buffers(void **state)
     instance->end_chart_formatting = NULL;
     instance->end_host_formatting = NULL;
     instance->end_batch_formatting = NULL;
-    assert_int_equal(__real_prepare_buffers(engine), 0);
+    __real_prepare_buffers(engine);
 
     assert_int_equal(instance->scheduled, 0);
     assert_int_equal(instance->after, 2);
@@ -705,7 +705,7 @@ static void test_format_host_labels_json_plaintext(void **state)
     instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
     assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0);
-    assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
+    assert_string_equal(buffer_tostring(instance->labels_buffer), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
 }
 
 static void test_format_host_labels_graphite_plaintext(void **state)
@@ -717,7 +717,7 @@ static void test_format_host_labels_graphite_plaintext(void **state)
     instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
     assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0);
-    assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2");
+    assert_string_equal(buffer_tostring(instance->labels_buffer), ";key1=value1;key2=value2");
 }
 
 static void test_format_host_labels_opentsdb_telnet(void **state)
@@ -729,7 +729,7 @@ static void test_format_host_labels_opentsdb_telnet(void **state)
     instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
     assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0);
-    assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2");
+    assert_string_equal(buffer_tostring(instance->labels_buffer), " key1=value1 key2=value2");
 }
 
 static void test_format_host_labels_opentsdb_http(void **state)
@@ -741,7 +741,7 @@ static void test_format_host_labels_opentsdb_http(void **state)
     instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
     assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0);
-    assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\"");
+    assert_string_equal(buffer_tostring(instance->labels_buffer), ",\"key1