track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

2 files changed, 17 insertions, 3 deletions

diff --git a/libnetdata/worker_utilization/worker_utilization.c b/libnetdata/worker_utilization/worker_utilization.c
index b0dcf034dd..867cd266c6 100644
--- a/libnetdata/worker_utilization/worker_utilization.c
+++ b/libnetdata/worker_utilization/worker_utilization.c
@@ -52,6 +52,7 @@ struct workers_workname {                           // this is what we add to Ju
 static struct workers_globals {
     SPINLOCK spinlock;
     Pvoid_t worknames_JudyHS;
+    size_t memory;
 
 } workers_globals = {                               // workers globals, the base of all worknames
         .spinlock = NETDATA_SPINLOCK_INITIALIZER,   // a lock for the worknames index
@@ -60,6 +61,14 @@ static struct workers_globals {
 
 static __thread struct worker *worker = NULL; // the current thread worker
 
+size_t workers_allocated_memory(void) {
+    netdata_spinlock_lock(&workers_globals.spinlock);
+    size_t memory = workers_globals.memory;
+    netdata_spinlock_unlock(&workers_globals.spinlock);
+
+    return memory;
+}
+
 void worker_register(const char *name) {
     if(unlikely(worker)) return;
 
@@ -76,9 +85,9 @@ void worker_register(const char *name) {
     size_t name_size = strlen(name) + 1;
     netdata_spinlock_lock(&workers_globals.spinlock);
 
-    Pvoid_t *PValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)name, name_size);
-    if(!PValue)
-        PValue = JudyHSIns(&workers_globals.worknames_JudyHS, (void *)name, name_size, PJE0);
+    workers_globals.memory += sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;
+
+    Pvoid_t *PValue = JudyHSIns(&workers_globals.worknames_JudyHS, (void *)name, name_size, PJE0);
 
     struct workers_workname *workname = *PValue;
     if(!workname) {
@@ -86,6 +95,8 @@ void worker_register(const char *name) {
         netdata_spinlock_init(&workname->spinlock);
         workname->base = NULL;
         *PValue = workname;
+
+        workers_globals.memory += sizeof(struct workers_workname) + JUDYHS_INDEX_SIZE_ESTIMATE(name_size);
     }
 
     netdata_spinlock_lock(&workname->spinlock);
@@ -136,8 +147,10 @@ void worker_unregister(void) {
         if(!workname->base) {
             JudyHSDel(&workers_globals.worknames_JudyHS, (void *) worker->workname, workname_size, PJE0);
             freez(workname);
+            workers_globals.memory -= sizeof(struct workers_workname) + JUDYHS_INDEX_SIZE_ESTIMATE(workname_size);
         }
     }
+    workers_globals.memory -= sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;
     netdata_spinlock_unlock(&workers_globals.spinlock);
 
     for(int i  = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
diff --git a/libnetdata/worker_utilization/worker_utilization.h b/libnetdata/worker_utilization/worker_utilization.h
index f1412e6b49..6745a010bc 100644
--- a/libnetdata/worker_utilization/worker_utilization.h
+++ b/libnetdata/worker_utilization/worker_utilization.h
@@ -15,6 +15,7 @@ typedef enum {
     WORKER_METRIC_INCREMENTAL_TOTAL = 4,
 } WORKER_METRIC_TYPE;
 
+size_t workers_allocated_memory(void);
 void worker_register(const char *name);
 void worker_register_job_name(size_t job_id, const char *name);
 void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type);