track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

6 files changed, 16 insertions, 14 deletions

diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index efb56d2b28..ca0f4b7a03 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -913,7 +913,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
         if(!strcmp(name, "chart")) chart = value;
         else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
             if(!dimensions)
-                dimensions = buffer_create(100);
+                dimensions = buffer_create(100, &netdata_buffers_statistics.buffers_api);
 
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
diff --git a/web/api/health/health_cmdapi.c b/web/api/health/health_cmdapi.c
index bad3e960a8..7a939bc0f5 100644
--- a/web/api/health/health_cmdapi.c
+++ b/web/api/health/health_cmdapi.c
@@ -196,7 +196,7 @@ int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, c
     w->response.data = wb;
     buffer_no_cacheable(w->response.data);
     if (ret == HTTP_RESP_OK && config_changed) {
-        BUFFER *jsonb = buffer_create(200);
+        BUFFER *jsonb = buffer_create(200, &netdata_buffers_statistics.buffers_health);
         health_silencers2json(jsonb);
         health_silencers2file(jsonb);
         buffer_free(jsonb);
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 872868fd3f..e5dcd79719 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -1550,7 +1550,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
     if(unlikely(tier >= storage_tiers)) return;
     if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
 
-    struct rrddim_tier *t = rd->tiers[tier];
+    struct rrddim_tier *t = &rd->tiers[tier];
     if(unlikely(!t)) return;
 
     time_t latest_time_s = t->query_ops->latest_time_s(t->db_metric_handle);
@@ -1567,14 +1567,14 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s
 
     // for each lower tier
     for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){
-        time_t smaller_tier_first_time = rd->tiers[read_tier]->query_ops->oldest_time_s(rd->tiers[read_tier]->db_metric_handle);
-        time_t smaller_tier_last_time = rd->tiers[read_tier]->query_ops->latest_time_s(rd->tiers[read_tier]->db_metric_handle);
+        time_t smaller_tier_first_time = rd->tiers[read_tier].query_ops->oldest_time_s(rd->tiers[read_tier].db_metric_handle);
+        time_t smaller_tier_last_time = rd->tiers[read_tier].query_ops->latest_time_s(rd->tiers[read_tier].db_metric_handle);
         if(smaller_tier_last_time <= latest_time_s) continue;  // it is as bad as we are
 
         long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s;
         long before_wanted = smaller_tier_last_time;
 
-        struct rrddim_tier *tmp = rd->tiers[read_tier];
+        struct rrddim_tier *tmp = &rd->tiers[read_tier];
         tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH);
 
         size_t points_read = 0;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index a4d4836221..1b38a33b1b 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -312,7 +312,7 @@ inline int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_clien
             else if (!strcmp("CLEAR", value)) status = RRDCALC_STATUS_CLEAR;
         }
         else if(!strcmp(name, "context") || !strcmp(name, "ctx")) {
-            if(!contexts) contexts = buffer_create(255);
+            if(!contexts) contexts = buffer_create(255, &netdata_buffers_statistics.buffers_api);
             buffer_strcat(contexts, "|");
             buffer_strcat(contexts, value);
         }
@@ -460,7 +460,7 @@ static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w
         else if(!strcmp(name, "chart_label_key")) chart_label_key = value;
         else if(!strcmp(name, "chart_labels_filter")) chart_labels_filter = value;
         else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
-            if(!dimensions) dimensions = buffer_create(100);
+            if(!dimensions) dimensions = buffer_create(100, &netdata_buffers_statistics.buffers_api);
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
         }
@@ -521,7 +521,7 @@ static int web_client_api_request_v1_contexts(RRDHOST *host, struct web_client *
         else if(!strcmp(name, "chart_label_key")) chart_label_key = value;
         else if(!strcmp(name, "chart_labels_filter")) chart_labels_filter = value;
         else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
-            if(!dimensions) dimensions = buffer_create(100);
+            if(!dimensions) dimensions = buffer_create(100, &netdata_buffers_statistics.buffers_api);
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
         }
@@ -626,7 +626,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         else if(!strcmp(name, "chart_labels_filter")) chart_labels_filter = value;
         else if(!strcmp(name, "chart")) chart = value;
         else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
-            if(!dimensions) dimensions = buffer_create(100);
+            if(!dimensions) dimensions = buffer_create(100, &netdata_buffers_statistics.buffers_api);
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
         }
diff --git a/web/server/web_client_cache.c b/web/server/web_client_cache.c
index 17d411157c..4344209c8a 100644
--- a/web/server/web_client_cache.c
+++ b/web/server/web_client_cache.c
@@ -65,13 +65,15 @@ static void web_client_free(struct web_client *w) {
     }
 #endif
     freez(w);
+    __atomic_sub_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
 }
 
 static struct web_client *web_client_alloc(void) {
     struct web_client *w = callocz(1, sizeof(struct web_client));
-    w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
-    w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
-    w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
+    __atomic_add_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
+    w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
+    w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
+    w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
     return w;
 }
 
diff --git a/web/server/web_server.c b/web/server/web_server.c
index 4da08d4319..d5645a947e 100644
--- a/web/server/web_server.c
+++ b/web/server/web_server.c
@@ -37,7 +37,7 @@ LISTEN_SOCKETS api_sockets = {
 };
 
 void debug_sockets() {
-	BUFFER *wb = buffer_create(256 * sizeof(char));
+	BUFFER *wb = buffer_create(256 * sizeof(char), NULL);
 	int i;
 
 	for(i = 0 ; i < (int)api_sockets.opened ; i++) {