track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

10 files changed, 24 insertions, 24 deletions

diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 0b33f6428c..f1964f3e57 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -48,7 +48,7 @@ int init_graphite_instance(struct instance *instance)
 
     instance->check_response = exporting_discard_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     if (!instance->buffer) {
         error("EXPORTING: cannot create buffer for graphite exporting connector instance %s", instance->config.name);
         return 1;
@@ -96,7 +96,7 @@ void sanitize_graphite_label_value(char *dst, const char *src, size_t len)
 int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
 {
     if (!instance->labels_buffer)
-        instance->labels_buffer = buffer_create(1024);
+        instance->labels_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_exporters);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c
index bfb6525ea6..15e1951f8f 100644
--- a/exporting/init_connectors.c
+++ b/exporting/init_connectors.c
@@ -171,8 +171,8 @@ void simple_connector_init(struct instance *instance)
     if (connector_specific_data->first_buffer)
         return;
 
-    connector_specific_data->header = buffer_create(0);
-    connector_specific_data->buffer = buffer_create(0);
+    connector_specific_data->header = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
+    connector_specific_data->buffer = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
     // create a ring buffer
     struct simple_connector_buffer *first_buffer = NULL;
@@ -195,7 +195,7 @@ void simple_connector_init(struct instance *instance)
     connector_specific_data->last_buffer = connector_specific_data->first_buffer;
 
     if (*instance->config.username || *instance->config.password) {
-        BUFFER *auth_string = buffer_create(0);
+        BUFFER *auth_string = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
         buffer_sprintf(auth_string, "%s:%s", instance->config.username, instance->config.password);
 
diff --git a/exporting/json/json.c b/exporting/json/json.c
index dd53f6f0a4..4cafd4c048 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -37,7 +37,7 @@ int init_json_instance(struct instance *instance)
 
     instance->check_response = exporting_discard_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     if (!instance->buffer) {
         error("EXPORTING: cannot create buffer for json exporting connector instance %s", instance->config.name);
         return 1;
@@ -96,7 +96,7 @@ int init_json_http_instance(struct instance *instance)
 
     instance->check_response = exporting_discard_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
     simple_connector_init(instance);
 
@@ -119,7 +119,7 @@ int init_json_http_instance(struct instance *instance)
 int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
 {
     if (!instance->labels_buffer)
-        instance->labels_buffer = buffer_create(1024);
+        instance->labels_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_exporters);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
diff --git a/exporting/mongodb/mongodb.c b/exporting/mongodb/mongodb.c
index 850d07fb30..186a7dcfdc 100644
--- a/exporting/mongodb/mongodb.c
+++ b/exporting/mongodb/mongodb.c
@@ -106,7 +106,7 @@ int init_mongodb_instance(struct instance *instance)
     instance->prepare_header = NULL;
     instance->check_response = NULL;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     if (!instance->buffer) {
         error("EXPORTING: cannot create buffer for MongoDB exporting connector instance %s", instance->config.name);
         return 1;
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index a974c12645..fc01ae4611 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -45,7 +45,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)
     instance->prepare_header = NULL;
     instance->check_response = exporting_discard_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     if (!instance->buffer) {
         error("EXPORTING: cannot create buffer for opentsdb telnet exporting connector instance %s", instance->config.name);
         return 1;
@@ -102,7 +102,7 @@ int init_opentsdb_http_instance(struct instance *instance)
     instance->prepare_header = opentsdb_http_prepare_header;
     instance->check_response = exporting_discard_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     if (!instance->buffer) {
         error("EXPORTING: cannot create buffer for opentsdb HTTP exporting connector instance %s", instance->config.name);
         return 1;
@@ -150,7 +150,7 @@ void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len)
 
 int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) {
     if(!instance->labels_buffer)
-        instance->labels_buffer = buffer_create(1024);
+        instance->labels_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_exporters);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
@@ -283,7 +283,7 @@ void opentsdb_http_prepare_header(struct instance *instance)
 
 int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) {
     if (!instance->labels_buffer)
-        instance->labels_buffer = buffer_create(1024);
+        instance->labels_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_exporters);
 
     if (unlikely(!sending_labels_configured(instance)))
         return 0;
diff --git a/exporting/process_data.c b/exporting/process_data.c
index 7146e7e8db..c07afc738e 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -77,8 +77,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
     time_t before = instance->before;
 
     // find the edges of the rrd database for this chart
-    time_t first_t = rd->tiers[0]->query_ops->oldest_time_s(rd->tiers[0]->db_metric_handle);
-    time_t last_t = rd->tiers[0]->query_ops->latest_time_s(rd->tiers[0]->db_metric_handle);
+    time_t first_t = rd->tiers[0].query_ops->oldest_time_s(rd->tiers[0].db_metric_handle);
+    time_t last_t = rd->tiers[0].query_ops->latest_time_s(rd->tiers[0].db_metric_handle);
     time_t update_every = st->update_every;
     struct storage_engine_query_handle handle;
 
@@ -126,8 +126,8 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
     size_t counter = 0;
     NETDATA_DOUBLE sum = 0;
 
-    for (rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, after, before, STORAGE_PRIORITY_LOW); !rd->tiers[0]->query_ops->is_finished(&handle);) {
-        STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle);
+    for (rd->tiers[0].query_ops->init(rd->tiers[0].db_metric_handle, &handle, after, before, STORAGE_PRIORITY_LOW); !rd->tiers[0].query_ops->is_finished(&handle);) {
+        STORAGE_POINT sp = rd->tiers[0].query_ops->next_metric(&handle);
         points_read++;
 
         if (unlikely(storage_point_is_empty(sp))) {
@@ -138,7 +138,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
         sum += sp.sum;
         counter += sp.count;
     }
-    rd->tiers[0]->query_ops->finalize(&handle);
+    rd->tiers[0].query_ops->finalize(&handle);
     global_statistics_exporters_query_completed(points_read);
 
     if (unlikely(!counter)) {
@@ -397,7 +397,7 @@ int simple_connector_end_batch(struct instance *instance)
     struct simple_connector_buffer *last_buffer = simple_connector_data->last_buffer;
 
     if (!last_buffer->buffer) {
-        last_buffer->buffer = buffer_create(0);
+        last_buffer->buffer = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
     }
 
     if (last_buffer->used) {
@@ -419,7 +419,7 @@ int simple_connector_end_batch(struct instance *instance)
     if (last_buffer->header)
         buffer_flush(last_buffer->header);
     else
-        last_buffer->header = buffer_create(0);
+        last_buffer->header = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
     if (instance->prepare_header)
         instance->prepare_header(instance);
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index 294d8ec2c9..dc675dd326 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -317,7 +317,7 @@ void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
         return;
 
     if (!instance->labels_buffer)
-        instance->labels_buffer = buffer_create(1024);
+        instance->labels_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_exporters);
 
     struct format_prometheus_label_callback tmp = {
         .instance = instance,
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index 2e2fa3c129..1857ca3338 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -104,7 +104,7 @@ int init_prometheus_remote_write_instance(struct instance *instance)
     instance->prepare_header = prometheus_remote_write_prepare_header;
     instance->check_response = process_prometheus_remote_write_response;
 
-    instance->buffer = (void *)buffer_create(0);
+    instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
     if (uv_mutex_init(&instance->mutex))
         return 1;
diff --git a/exporting/send_data.c b/exporting/send_data.c
index 1d20f3b748..045aab6ed6 100644
--- a/exporting/send_data.c
+++ b/exporting/send_data.c
@@ -64,7 +64,7 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
 {
     static BUFFER *response = NULL;
     if (!response)
-        response = buffer_create(4096);
+        response = buffer_create(4096, &netdata_buffers_statistics.buffers_exporters);
 
     struct stats *stats = &instance->stats;
 #ifdef ENABLE_HTTPS
diff --git a/exporting/send_internal_metrics.c b/exporting/send_internal_metrics.c
index 515cda3b29..e4347964f0 100644
--- a/exporting/send_internal_metrics.c
+++ b/exporting/send_internal_metrics.c
@@ -65,7 +65,7 @@ void send_internal_metrics(struct instance *instance)
 
     if (!stats->initialized) {
         char id[RRD_ID_LENGTH_MAX + 1];
-        BUFFER *family = buffer_create(0);
+        BUFFER *family = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
 
         buffer_sprintf(family, "exporting_%s", instance->config.name);