track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

2 files changed, 36 insertions, 27 deletions

diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c
index d0940588fc..eeb283209f 100644
--- a/libnetdata/buffer/buffer.c
+++ b/libnetdata/buffer/buffer.c
@@ -442,28 +442,28 @@ void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minute
     buffer_need_bytes(wb, 36);
 
     char *b = &wb->buffer[wb->len];
-  char *p = b;
-
-  *p++ = '0' + year / 1000; year %= 1000;
-  *p++ = '0' + year / 100;  year %= 100;
-  *p++ = '0' + year / 10;
-  *p++ = '0' + year % 10;
-  *p++ = '-';
-  *p++ = '0' + month / 10;
-  *p++ = '0' + month % 10;
-  *p++ = '-';
-  *p++ = '0' + day / 10;
-  *p++ = '0' + day % 10;
-  *p++ = ' ';
-  *p++ = '0' + hours / 10;
-  *p++ = '0' + hours % 10;
-  *p++ = ':';
-  *p++ = '0' + minutes / 10;
-  *p++ = '0' + minutes % 10;
-  *p++ = ':';
-  *p++ = '0' + seconds / 10;
-  *p++ = '0' + seconds % 10;
-  *p = '\0';
+    char *p = b;
+
+    *p++ = '0' + year / 1000; year %= 1000;
+    *p++ = '0' + year / 100;  year %= 100;
+    *p++ = '0' + year / 10;
+    *p++ = '0' + year % 10;
+    *p++ = '-';
+    *p++ = '0' + month / 10;
+    *p++ = '0' + month % 10;
+    *p++ = '-';
+    *p++ = '0' + day / 10;
+    *p++ = '0' + day % 10;
+    *p++ = ' ';
+    *p++ = '0' + hours / 10;
+    *p++ = '0' + hours % 10;
+    *p++ = ':';
+    *p++ = '0' + minutes / 10;
+    *p++ = '0' + minutes % 10;
+    *p++ = ':';
+    *p++ = '0' + seconds / 10;
+    *p++ = '0' + seconds % 10;
+    *p = '\0';
 
     wb->len += (size_t)(p - b);
 
@@ -472,7 +472,7 @@ void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minute
     buffer_overflow_check(wb);
 }
 
-BUFFER *buffer_create(size_t size)
+BUFFER *buffer_create(size_t size, size_t *statistics)
 {
     BUFFER *b;
 
@@ -483,9 +483,13 @@ BUFFER *buffer_create(size_t size)
     b->buffer[0] = '\0';
     b->size = size;
     b->contenttype = CT_TEXT_PLAIN;
+    b->statistics = statistics;
     buffer_overflow_init(b);
     buffer_overflow_check(b);
 
+    if(b->statistics)
+        __atomic_add_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED);
+
     return(b);
 }
 
@@ -496,6 +500,9 @@ void buffer_free(BUFFER *b) {
 
     debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size);
 
+    if(b->statistics)
+        __atomic_sub_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED);
+
     freez(b->buffer);
     freez(b);
 }
@@ -510,9 +517,7 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
     size_t minimum = WEB_DATA_LENGTH_INCREASE_STEP;
     if(minimum > wanted) wanted = minimum;
 
-    size_t optimal = b->size;
-    if(b->size > 5*1024*1024) optimal = b->size / 2;
-
+    size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size;
     if(optimal > wanted) wanted = optimal;
 
     debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted);
@@ -520,6 +525,9 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
     b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2);
     b->size += wanted;
 
+    if(b->statistics)
+        __atomic_add_fetch(b->statistics, wanted, __ATOMIC_RELAXED);
+
     buffer_overflow_init(b);
     buffer_overflow_check(b);
 }
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index ce6f52899b..0fa3495b4a 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -15,6 +15,7 @@ typedef struct web_buffer {
     uint8_t options;		// options related to the content
     time_t date;    		// the timestamp this content has been generated
     time_t expires;			// the timestamp this content expires
+    size_t *statistics;
 } BUFFER;
 
 // options
@@ -61,7 +62,7 @@ void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value);
 void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
 void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
 
-BUFFER *buffer_create(size_t size);
+BUFFER *buffer_create(size_t size, size_t *statistics);
 void buffer_free(BUFFER *b);
 void buffer_increase(BUFFER *b, size_t free_size_required);