track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 25 insertions, 10 deletions

diff --git a/libnetdata/string/string.c b/libnetdata/string/string.c
index d2db8aab43..4e232523c8 100644
--- a/libnetdata/string/string.c
+++ b/libnetdata/string/string.c
@@ -56,14 +56,29 @@ static struct string_hashtable {
 #define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED)
 
 void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) {
-    *inserts = string_base.inserts;
-    *deletes = string_base.deletes;
-    *searches = string_base.searches;
-    *entries = (size_t)string_base.entries;
-    *references = (size_t)string_base.active_references;
-    *memory = (size_t)string_base.memory;
-    *duplications = string_base.duplications;
-    *releases = string_base.releases;
+    if(inserts)
+        *inserts = string_base.inserts;
+
+    if(deletes)
+        *deletes = string_base.deletes;
+
+    if(searches)
+        *searches = string_base.searches;
+
+    if(entries)
+        *entries = (size_t)string_base.entries;
+
+    if(references)
+        *references = (size_t)string_base.active_references;
+
+    if(memory)
+        *memory = (size_t)string_base.memory;
+
+    if(duplications)
+        *duplications = string_base.duplications;
+
+    if(releases)
+        *releases = string_base.releases;
 }
 
 #define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);
@@ -186,7 +201,7 @@ static inline STRING *string_index_insert(const char *str, size_t length) {
         *ptr = string;
         string_base.inserts++;
         string_base.entries++;
-        string_base.memory += (long)mem_size;
+        string_base.memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length));
     }
     else {
         // the item is already in the index
@@ -240,7 +255,7 @@ static inline void string_index_delete(STRING *string) {
         size_t mem_size = sizeof(STRING) + string->length;
         string_base.deletes++;
         string_base.entries--;
-        string_base.memory -= (long)mem_size;
+        string_base.memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length));
         freez(string);
     }