track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

7 files changed, 271 insertions, 69 deletions

diff --git a/daemon/analytics.c b/daemon/analytics.c
index dd7eeebbd7..a2f52bc8f6 100644
--- a/daemon/analytics.c
+++ b/daemon/analytics.c
@@ -241,7 +241,7 @@ void analytics_exporters(void)
 {
     //when no exporters are available, an empty string will be sent
     //decide if something else is more suitable (but probably not null)
-    BUFFER *bi = buffer_create(1000);
+    BUFFER *bi = buffer_create(1000, NULL);
     analytics_exporting_connectors(bi);
     analytics_set_data_str(&analytics_data.netdata_exporting_connectors, (char *)buffer_tostring(bi));
     buffer_free(bi);
@@ -278,7 +278,7 @@ void analytics_collectors(void)
     RRDSET *st;
     DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
     char name[500];
-    BUFFER *bt = buffer_create(1000);
+    BUFFER *bt = buffer_create(1000, NULL);
 
     rrdset_foreach_read(st, localhost) {
         if(!rrdset_is_available_for_viewers(st))
@@ -333,7 +333,7 @@ void analytics_alarms_notifications(void)
 
     debug(D_ANALYTICS, "Executing %s", script);
 
-    BUFFER *b = buffer_create(1000);
+    BUFFER *b = buffer_create(1000, NULL);
     int cnt = 0;
     FILE *fp_child_input;
     FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input);
@@ -380,7 +380,7 @@ void analytics_get_install_type(void)
  */
 void analytics_https(void)
 {
-    BUFFER *b = buffer_create(30);
+    BUFFER *b = buffer_create(30, NULL);
 #ifdef ENABLE_HTTPS
     analytics_exporting_connectors_ssl(b);
     buffer_strcat(b, netdata_ssl_client_ctx && rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) && localhost->sender->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ? "streaming|" : "|");
@@ -675,7 +675,7 @@ void set_late_global_environment()
     analytics_set_data_str(&analytics_data.netdata_config_release_channel, (char *)get_release_channel());
 
     {
-        BUFFER *bi = buffer_create(1000);
+        BUFFER *bi = buffer_create(1000, NULL);
         analytics_build_info(bi);
         analytics_set_data_str(&analytics_data.netdata_buildinfo, (char *)buffer_tostring(bi));
         buffer_free(bi);
@@ -836,7 +836,7 @@ void set_global_environment()
     setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1);
 
     {
-        BUFFER *user_plugins_dirs = buffer_create(FILENAME_MAX);
+        BUFFER *user_plugins_dirs = buffer_create(FILENAME_MAX, NULL);
 
         for (size_t i = 1; i < PLUGINSD_MAX_DIRECTORIES && plugin_directories[i]; i++) {
             if (i > 1)
diff --git a/daemon/commands.c b/daemon/commands.c
index 8f09e63e11..377a4002f3 100644
--- a/daemon/commands.c
+++ b/daemon/commands.c
@@ -220,7 +220,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
     info("COMMAND: reloading host labels.");
     reload_host_labels();
 
-    BUFFER *wb = buffer_create(10);
+    BUFFER *wb = buffer_create(10, NULL);
     rrdlabels_log_to_buffer(localhost->rrdlabels, wb);
     (*message)=strdupz(buffer_tostring(wb));
     buffer_free(wb);
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 3af4368c48..2422b5a3a4 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -20,6 +20,11 @@
 
 bool global_statistics_enabled = true;
 
+struct netdata_buffers_statistics netdata_buffers_statistics = {};
+
+static size_t dbengine_total_memory = 0;
+size_t rrddim_db_memory_size = 0;
+
 static struct global_statistics {
     uint16_t connected_clients;
 
@@ -214,6 +219,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
     }
 }
 
+#define dictionary_stats_memory_total(stats) \
+    ((stats).memory.dict + (stats).memory.values + (stats).memory.index)
+
 static void global_statistics_charts(void) {
     static unsigned long long old_web_requests = 0,
                               old_web_usec = 0,
@@ -270,23 +278,168 @@ static void global_statistics_charts(void) {
     // ----------------------------------------------------------------
 
     {
-        static RRDSET *st_uptime = NULL;
-        static RRDDIM *rd_uptime = NULL;
+        static RRDSET *st_memory = NULL;
+        static RRDDIM *rd_database = NULL;
+        static RRDDIM *rd_collectors = NULL;
+        static RRDDIM *rd_hosts = NULL;
+        static RRDDIM *rd_rrd = NULL;
+        static RRDDIM *rd_contexts = NULL;
+        static RRDDIM *rd_health = NULL;
+        static RRDDIM *rd_functions = NULL;
+        static RRDDIM *rd_labels = NULL;
+        static RRDDIM *rd_strings = NULL;
+        static RRDDIM *rd_streaming = NULL;
+        static RRDDIM *rd_replication = NULL;
+        static RRDDIM *rd_buffers = NULL;
+        static RRDDIM *rd_workers = NULL;
+        static RRDDIM *rd_other = NULL;
+
+        if (unlikely(!st_memory)) {
+            st_memory = rrdset_create_localhost(
+                    "netdata",
+                    "memory",
+                    NULL,
+                    "netdata",
+                    NULL,
+                    "Netdata Memory",
+                    "bytes",
+                    "netdata",
+                    "stats",
+                    130100,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_STACKED);
 
-        if (unlikely(!st_uptime)) {
-            st_uptime = rrdset_create_localhost(
+            rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_hosts = rrddim_add(st_memory, "hosts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_rrd = rrddim_add(st_memory, "rrdset rrddim", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_contexts = rrddim_add(st_memory, "contexts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_health = rrddim_add(st_memory, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_functions = rrddim_add(st_memory, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_labels = rrddim_add(st_memory, "labels", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_strings = rrddim_add(st_memory, "strings", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_streaming = rrddim_add(st_memory, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+        }
+
+        size_t buffers =
+            netdata_buffers_statistics.query_targets_size +
+            netdata_buffers_statistics.rrdset_done_rda_size +
+            netdata_buffers_statistics.buffers_aclk +
+            netdata_buffers_statistics.buffers_api +
+            netdata_buffers_statistics.buffers_functions +
+            netdata_buffers_statistics.buffers_sqlite +
+            netdata_buffers_statistics.buffers_exporters +
+            netdata_buffers_statistics.buffers_health +
+            netdata_buffers_statistics.buffers_streaming +
+            netdata_buffers_statistics.cbuffers_streaming +
+            netdata_buffers_statistics.buffers_web +
+            replication_allocated_buffers();
+
+        size_t strings = 0;
+        string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL);
+
+        rrddim_set_by_pointer(st_memory, rd_database, (collected_number)dbengine_total_memory + (collected_number)rrddim_db_memory_size);
+        rrddim_set_by_pointer(st_memory, rd_collectors, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_collectors));
+        rrddim_set_by_pointer(st_memory, rd_hosts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhost) + (collected_number)netdata_buffers_statistics.rrdhost_allocations_size);
+        rrddim_set_by_pointer(st_memory, rd_rrd, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdset_rrddim));
+        rrddim_set_by_pointer(st_memory, rd_contexts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdcontext));
+        rrddim_set_by_pointer(st_memory, rd_health, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhealth));
+        rrddim_set_by_pointer(st_memory, rd_functions, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions));
+        rrddim_set_by_pointer(st_memory, rd_labels, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdlabels));
+        rrddim_set_by_pointer(st_memory, rd_strings, (collected_number)strings);
+        rrddim_set_by_pointer(st_memory, rd_streaming, (collected_number)netdata_buffers_statistics.rrdhost_senders + (collected_number)netdata_buffers_statistics.rrdhost_receivers);
+        rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory());
+        rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers);
+        rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory());
+        rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other));
+
+        rrdset_done(st_memory);
+    }
+
+    {
+        static RRDSET *st_memory_buffers = NULL;
+        static RRDDIM *rd_queries = NULL;
+        static RRDDIM *rd_collectors = NULL;
+        static RRDDIM *rd_buffers_aclk = NULL;
+        static RRDDIM *rd_buffers_api = NULL;
+        static RRDDIM *rd_buffers_functions = NULL;
+        static RRDDIM *rd_buffers_sqlite = NULL;
+        static RRDDIM *rd_buffers_exporters = NULL;
+        static RRDDIM *rd_buffers_health = NULL;
+        static RRDDIM *rd_buffers_streaming = NULL;
+        static RRDDIM *rd_cbuffers_streaming = NULL;
+        static RRDDIM *rd_buffers_replication = NULL;
+        static RRDDIM *rd_buffers_web = NULL;
+
+        if (unlikely(!st_memory_buffers)) {
+            st_memory_buffers = rrdset_create_localhost(
                 "netdata",
-                "uptime",
+                "memory_buffers",
                 NULL,
                 "netdata",
                 NULL,
-                "Netdata uptime",
-                "seconds",
+                "Netdata Memory Buffers",
+                "bytes",
                 "netdata",
                 "stats",
-                130100,
+                130101,
                 localhost->rrd_update_every,
-                RRDSET_TYPE_LINE);
+                RRDSET_TYPE_STACKED);
+
+            rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_collectors = rrddim_add(st_memory_buffers, "collection", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_aclk = rrddim_add(st_memory_buffers, "aclk", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_api = rrddim_add(st_memory_buffers, "api", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_functions = rrddim_add(st_memory_buffers, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_sqlite = rrddim_add(st_memory_buffers, "sqlite", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_exporters = rrddim_add(st_memory_buffers, "exporters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_health = rrddim_add(st_memory_buffers, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_streaming = rrddim_add(st_memory_buffers, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+        }
+
+        rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory());
+        rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_api, (collected_number)netdata_buffers_statistics.buffers_api);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_functions, (collected_number)netdata_buffers_statistics.buffers_functions);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_sqlite, (collected_number)netdata_buffers_statistics.buffers_sqlite);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_exporters, (collected_number)netdata_buffers_statistics.buffers_exporters);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming);
+        rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming);
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers());
+        rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web);
+
+        rrdset_done(st_memory_buffers);
+    }
+
+    // ----------------------------------------------------------------
+
+    {
+        static RRDSET *st_uptime = NULL;
+        static RRDDIM *rd_uptime = NULL;
+
+        if (unlikely(!st_uptime)) {
+            st_uptime = rrdset_create_localhost(
+                    "netdata",
+                    "uptime",
+                    NULL,
+                    "netdata",
+                    NULL,
+                    "Netdata uptime",
+                    "seconds",
+                    "netdata",
+                    "stats",
+                    130150,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_LINE);
 
             rd_uptime = rrddim_add(st_uptime, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
         }
@@ -1076,13 +1229,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_cache_hit_ratio)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_hit_ratio", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Hit Ratio", name);
 
             ptrs->st_cache_hit_ratio = rrdset_create_localhost(
@@ -1124,13 +1277,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_operations)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_operations", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Operations", name);
 
             ptrs->st_operations = rrdset_create_localhost(
@@ -1178,13 +1331,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_memory)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_memory", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Memory", name);
 
             ptrs->st_pgc_memory = rrdset_create_localhost(
@@ -1232,13 +1385,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_tm)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_target_memory", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Target Cache Memory", name);
 
             ptrs->st_pgc_tm = rrdset_create_localhost(
@@ -1282,13 +1435,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_pages)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_pages", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Pages", name);
 
             ptrs->st_pgc_pages = rrdset_create_localhost(
@@ -1326,13 +1479,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_memory_changes)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_memory_changes", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Memory Changes", name);
 
             ptrs->st_pgc_memory_changes = rrdset_create_localhost(
@@ -1368,13 +1521,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_memory_migrations)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_memory_migrations", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Memory Migrations", name);
 
             ptrs->st_pgc_memory_migrations = rrdset_create_localhost(
@@ -1408,13 +1561,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_memory_events)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_events", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Events", name);
 
             ptrs->st_pgc_memory_events = rrdset_create_localhost(
@@ -1450,13 +1603,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_waste)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_waste_events", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Waste Events", name);
 
             ptrs->st_pgc_waste = rrdset_create_localhost(
@@ -1502,13 +1655,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
 
     {
         if (unlikely(!ptrs->st_pgc_workers)) {
-            BUFFER *id = buffer_create(100);
+            BUFFER *id = buffer_create(100, NULL);
             buffer_sprintf(id, "dbengine_%s_cache_workers", name);
 
-            BUFFER *family = buffer_create(100);
+            BUFFER *family = buffer_create(100, NULL);
             buffer_sprintf(family, "dbengine %s cache", name);
 
-            BUFFER *title = buffer_create(100);
+            BUFFER *title = buffer_create(100, NULL);
             buffer_sprintf(title, "Netdata %s Cache Workers", name);
 
             ptrs->st_pgc_workers = rrdset_create_localhost(
@@ -1587,6 +1740,8 @@ static void dbengine2_statistics_charts(void) {
     buffers_total_size += buffers.julyl;
 #endif
 
+    dbengine_total_memory = pgc_main_stats.size + pgc_open_stats.size + pgc_extent_stats.size + mrg_stats.size + buffers_total_size;
+
     size_t priority = 135000;
 
     {
@@ -1620,6 +1775,7 @@ static void dbengine2_statistics_charts(void) {
         }
         priority++;
 
+
         rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_main, (collected_number)pgc_main_stats.size);
         rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_open, (collected_number)pgc_open_stats.size);
         rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_extent, (collected_number)pgc_extent_stats.size);
@@ -2549,6 +2705,15 @@ static void update_heartbeat_charts() {
 // ---------------------------------------------------------------------------------------------------------------------
 // dictionary statistics
 
+struct dictionary_stats dictionary_stats_category_collectors = { .name = "collectors" };
+struct dictionary_stats dictionary_stats_category_rrdhost = { .name = "rrdhost" };
+struct dictionary_stats dictionary_stats_category_rrdset_rrddim = { .name = "rrdset_rrddim" };
+struct dictionary_stats dictionary_stats_category_rrdcontext = { .name = "context" };
+struct dictionary_stats dictionary_stats_category_rrdlabels = { .name = "labels" };
+struct dictionary_stats dictionary_stats_category_rrdhealth = { .name = "health" };
+struct dictionary_stats dictionary_stats_category_functions = { .name = "functions" };
+struct dictionary_stats dictionary_stats_category_replication = { .name = "replication" };
+
 struct dictionary_categories {
     struct dictionary_stats *stats;
     const char *family;
@@ -2594,7 +2759,15 @@ struct dictionary_categories {
     RRDDIM *rd_spins_delete;
 
 } dictionary_categories[] = {
-    { .stats = &dictionary_stats_category_other, "dictionaries", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_collectors, "dictionaries collectors", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_rrdhost, "dictionaries hosts", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_rrdset_rrddim, "dictionaries rrd", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_rrdcontext, "dictionaries contexts", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_rrdlabels, "dictionaries labels", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_rrdhealth, "dictionaries health", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_functions, "dictionaries functions", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_replication, "dictionaries replication", "dictionaries", 900000 },
+    { .stats = &dictionary_stats_category_other, "dictionaries other", "dictionaries", 900000 },
 
     // terminator
     { .stats = NULL, NULL, NULL, 0 },
@@ -2804,7 +2977,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
     // ------------------------------------------------------------------------
 
     total = 0;
-    load_dictionary_stats_entry(memory.indexed);
+    load_dictionary_stats_entry(memory.index);
     load_dictionary_stats_entry(memory.values);
     load_dictionary_stats_entry(memory.dict);
 
@@ -2838,7 +3011,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
             rrdlabels_add(c->st_memory->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO);
         }
 
-        rrddim_set_by_pointer(c->st_memory, c->rd_memory_indexed, (collected_number)stats.memory.indexed);
+        rrddim_set_by_pointer(c->st_memory, c->rd_memory_indexed, (collected_number)stats.memory.index);
         rrddim_set_by_pointer(c->st_memory, c->rd_memory_values, (collected_number)stats.memory.values);
         rrddim_set_by_pointer(c->st_memory, c->rd_memory_dict, (collected_number)stats.memory.dict);
 
diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h
index c7f53781aa..7bdb153dd9 100644
--- a/daemon/global_statistics.h
+++ b/daemon/global_statistics.h
@@ -5,6 +5,34 @@
 
 #include "database/rrd.h"
 
+extern struct netdata_buffers_statistics {
+    size_t rrdhost_allocations_size;
+    size_t rrdhost_senders;
+    size_t rrdhost_receivers;
+    size_t query_targets_size;
+    size_t rrdset_done_rda_size;
+    size_t buffers_aclk;
+    size_t buffers_api;
+    size_t buffers_functions;
+    size_t buffers_sqlite;
+    size_t buffers_exporters;
+    size_t buffers_health;
+    size_t buffers_streaming;
+    size_t cbuffers_streaming;
+    size_t buffers_web;
+} netdata_buffers_statistics;
+
+extern struct dictionary_stats dictionary_stats_category_collectors;
+extern struct dictionary_stats dictionary_stats_category_rrdhost;
+extern struct dictionary_stats dictionary_stats_category_rrdset_rrddim;
+extern struct dictionary_stats dictionary_stats_category_rrdcontext;
+extern struct dictionary_stats dictionary_stats_category_rrdlabels;
+extern struct dictionary_stats dictionary_stats_category_rrdhealth;
+extern struct dictionary_stats dictionary_stats_category_functions;
+extern struct dictionary_stats dictionary_stats_category_replication;
+
+extern size_t rrddim_db_memory_size;
+
 // ----------------------------------------------------------------------------
 // global statistics
 
diff --git a/daemon/main.c b/daemon/main.c
index 408b7a2ded..2c10cb00bb 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -174,8 +174,8 @@ static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {
 }
 
 static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
-    BUFFER *service_list = buffer_create(1024);
-    BUFFER *thread_list = buffer_create(1024);
+    BUFFER *service_list = buffer_create(1024, NULL);
+    BUFFER *thread_list = buffer_create(1024, NULL);
     usec_t started_ut = now_monotonic_usec(), ended_ut;
     size_t running;
     SERVICE_TYPE running_services = 0;
@@ -1926,6 +1926,7 @@ int main(int argc, char **argv) {
 
     netdata_anonymous_statistics_enabled=-1;
     struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
+    __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);
     get_system_info(system_info);
     system_info->hops = 0;
     get_install_type(&system_info->install_type, &system_info->prebuilt_arch, &system_info->prebuilt_dist);
diff --git a/daemon/service.c b/daemon/service.c
index 3f5e8c55a3..9761abd02d 100644
--- a/daemon/service.c
+++ b/daemon/service.c
@@ -52,13 +52,13 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
 
         size_t tiers_available = 0, tiers_said_no_retention = 0;
         for(size_t tier = 0; tier < storage_tiers ;tier++) {
-            if(rd->tiers[tier]) {
+            if(rd->tiers[tier].db_collection_handle) {
                 tiers_available++;
 
-                if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
+                if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle))
                     tiers_said_no_retention++;
 
-                rd->tiers[tier]->db_collection_handle = NULL;
+                rd->tiers[tier].db_collection_handle = NULL;
             }
         }
 
diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index efb2d3cd9d..9845ed430f 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -439,7 +439,7 @@ int unit_test_str2ld() {
 }
 
 int unit_test_buffer() {
-    BUFFER *wb = buffer_create(1);
+    BUFFER *wb = buffer_create(1, NULL);
     char string[2048 + 1];
     char final[9000 + 1];
     int i;
@@ -1349,7 +1349,7 @@ static int test_variable_renames(void) {
     rrddim_reset_name(st, rd2, "DIM2NAME2");
     fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
 
-    BUFFER *buf = buffer_create(1);
+    BUFFER *buf = buffer_create(1, NULL);
     health_api_v1_chart_variables2json(st, buf);
     fprintf(stderr, "%s", buffer_tostring(buf));
     buffer_free(buf);
@@ -1604,7 +1604,7 @@ int test_sqlite(void) {
         return 1;
     }
 
-    BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE);
+    BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, NULL);
     char *uuid_str = "0000_000";
 
     buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
@@ -1861,7 +1861,7 @@ static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDI
     // Fluh pages for subsequent real values
     for (i = 0 ; i < CHARTS ; ++i) {
         for (j = 0; j < DIMS; ++j) {
-            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
+            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle);
         }
     }
 }
@@ -1880,7 +1880,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
     // feed it with the test data
     for (i = 0 ; i < CHARTS ; ++i) {
         for (j = 0 ; j < DIMS ; ++j) {
-            rd[i][j]->tiers[0]->collect_ops->change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
+            rd[i][j]->tiers[0].collect_ops->change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every);
 
             rd[i][j]->last_collected_time.tv_sec =
             st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
@@ -1931,13 +1931,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
         time_now = time_start + (c + 1) * update_every;
         for (i = 0 ; i < CHARTS ; ++i) {
             for (j = 0; j < DIMS; ++j) {
-                rd[i][j]->tiers[0]->query_ops->init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
+                rd[i][j]->tiers[0].query_ops->init(rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL);
                 for (k = 0; k < QUERY_BATCH; ++k) {
                     last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
                            j * REGION_POINTS[current_region] + c + k;
                     expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
 
-                    STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops->next_metric(&handle);
+                    STORAGE_POINT sp = rd[i][j]->tiers[0].query_ops->next_metric(&handle);
                     value = sp.sum;
                     time_retrieved = sp.start_time_s;
                     end_time = sp.end_time_s;
@@ -1959,7 +1959,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
                         errors++;
                     }
                 }
-                rd[i][j]->tiers[0]->query_ops->finalize(&handle);
+                rd[i][j]->tiers[0].query_ops->finalize(&handle);
             }
         }
     }
@@ -2084,7 +2084,7 @@ int test_dbengine(void)
     for (i = 0 ; i < CHARTS ; ++i) {
         st[i]->update_every = update_every;
         for (j = 0; j < DIMS; ++j) {
-            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
+            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle);
         }
     }
 
@@ -2103,7 +2103,7 @@ int test_dbengine(void)
     for (i = 0 ; i < CHARTS ; ++i) {
         st[i]->update_every = update_every;
         for (j = 0; j < DIMS; ++j) {
-            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle);
+            rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle);
         }
     }
 
@@ -2270,7 +2270,7 @@ static void generate_dbengine_chart(void *arg)
         thread_info->time_max = time_current;
     }
     for (j = 0; j < DSET_DIMS; ++j) {
-        rrdeng_store_metric_finalize((rd[j])->tiers[0]->db_collection_handle);
+        rrdeng_store_metric_finalize((rd[j])->tiers[0].db_collection_handle);
     }
 }
 
@@ -2390,13 +2390,13 @@ static void query_dbengine_chart(void *arg)
             time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
         }
 
-        rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
+        rd->tiers[0].query_ops->init(rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL);
         ++thread_info->queries_nr;
         for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
             generatedv = generate_dbengine_chart_value(i, j, time_now);
             expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
 
-            if (unlikely(rd->tiers[0]->query_ops->is_finished(&handle))) {
+