track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

2 files changed, 8 insertions, 6 deletions

diff --git a/health/health.c b/health/health.c
index 36fcd2f2b1..947ef8644d 100644
--- a/health/health.c
+++ b/health/health.c
@@ -459,8 +459,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
     BUFFER *warn_alarms, *crit_alarms;
     active_alerts_t *active_alerts = callocz(ACTIVE_ALARMS_LIST_EXAMINE, sizeof(active_alerts_t));
 
-    warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
-    crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
+    warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
+    crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_health);
 
     foreach_rrdcalc_in_rrdhost_read(host, rc) {
         if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
@@ -517,7 +517,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
 
     char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");
 
-    BUFFER *wb = buffer_create(8192);
+    BUFFER *wb = buffer_create(8192, &netdata_buffers_statistics.buffers_health);
     bool ok = prepare_command(wb,
                               exec,
                               recipient,
diff --git a/health/health_log.c b/health/health_log.c
index 676e78b0a4..b7e3e7ef49 100644
--- a/health/health_log.c
+++ b/health/health_log.c
@@ -73,7 +73,7 @@ inline void health_label_log_save(RRDHOST *host) {
     health_log_rotate(host);
 
     if(unlikely(host->health.health_log_fp)) {
-        BUFFER *wb = buffer_create(1024);
+        BUFFER *wb = buffer_create(1024, &netdata_buffers_statistics.buffers_health);
 
         rrdlabels_to_buffer(localhost->rrdlabels, wb, "", "=", "", "\t ", NULL, NULL, NULL, NULL);
         char *write = (char *) buffer_tostring(wb);
@@ -182,8 +182,10 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
     size_t line = 0, len = 0;
     ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0;
 
-    DICTIONARY *all_rrdcalcs = dictionary_create(
-        DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE);
+    DICTIONARY *all_rrdcalcs = dictionary_create_advanced(
+        DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
+        &dictionary_stats_category_rrdhealth);
+
     RRDCALC *rc;
     foreach_rrdcalc_in_rrdhost_read(host, rc) {
         dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc));