From 9232bfb6a072155388578dc4e1338c6002afb515 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Fri, 20 Jan 2023 00:50:42 +0200 Subject: track memory footprint of Netdata (#14294) * track memory footprint of Netdata * track db modes alloc/ram/save/map * track system info; track sender and receiver * fixes * more fixes * track workers memory, onewayalloc memory; unify judyhs size estimation * track replication structures and buffers * Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag * flush the replication buffer every 1000 times the circular buffer is found empty * dont take timestamp too frequently in sender loop * sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it * free sender thread buffer on replication threads when replication is idle * use the last sender flag as a timestamp of the last buffer recreation * free cbuffer before reconnecting * recreate cbuffer on every flush * timings for journal v2 loading * inlining of metric and cache functions * aral likely/unlikely * free left-over thread buffers * fix NULL pointer dereference in replication * free sender thread buffer on sender thread too * mark ctx as used before flushing * better logging on ctx datafiles closing Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> --- aclk/aclk.c | 2 +- aclk/aclk_otp.c | 8 +- aclk/aclk_query.c | 12 +- aclk/https_client.c | 4 +- collectors/apps.plugin/apps_plugin.c | 2 +- collectors/diskspace.plugin/plugin_diskspace.c | 2 +- collectors/plugins.d/pluginsd_parser.c | 2 +- collectors/proc.plugin/proc_self_mountinfo.c | 5 +- collectors/proc.plugin/proc_spl_kstat_zfs.c | 2 +- collectors/proc.plugin/sys_block_zram.c | 2 +- collectors/statsd.plugin/statsd.c | 20 +- collectors/tc.plugin/plugin_tc.c | 7 +- daemon/analytics.c | 12 +- daemon/commands.c | 2 +- daemon/global_statistics.c | 257 +++++++++++++++++---- daemon/global_statistics.h | 28 +++ daemon/main.c | 5 +- daemon/service.c | 6 +- daemon/unit_test.c | 30 +-- database/engine/cache.c | 43 ++-- database/engine/cache.h | 4 +- database/engine/datafile.c | 30 ++- database/engine/journalfile.c | 27 ++- database/engine/metric.c | 36 +-- database/engine/pagecache.c | 12 +- database/ram/rrddim_mem.c | 9 + database/rrd.h | 44 ++-- database/rrdcalc.c | 2 +- database/rrdcalctemplate.c | 2 +- database/rrdcontext.c | 91 +++++--- database/rrddim.c | 60 ++--- database/rrddimvar.c | 2 +- database/rrdfamily.c | 2 +- database/rrdfunctions.c | 4 +- database/rrdhost.c | 21 +- database/rrdlabels.c | 2 +- database/rrdset.c | 35 +-- database/rrdsetvar.c | 2 +- database/rrdvar.c | 2 +- database/sqlite/sqlite_aclk.c | 8 +- database/sqlite/sqlite_aclk_alert.c | 18 +- database/sqlite/sqlite_metadata.c | 7 +- exporting/graphite/graphite.c | 4 +- exporting/init_connectors.c | 6 +- exporting/json/json.c | 6 +- exporting/mongodb/mongodb.c | 2 +- exporting/opentsdb/opentsdb.c | 8 +- exporting/process_data.c | 14 +- exporting/prometheus/prometheus.c | 2 +- exporting/prometheus/remote_write/remote_write.c | 2 +- exporting/send_data.c | 2 +- exporting/send_internal_metrics.c | 2 +- health/health.c | 6 +- health/health_log.c | 8 +- libnetdata/arrayalloc/arrayalloc.c | 12 +- libnetdata/buffer/buffer.c | 60 ++--- libnetdata/buffer/buffer.h | 3 +- libnetdata/circular_buffer/circular_buffer.c | 30 ++- libnetdata/circular_buffer/circular_buffer.h | 3 +- libnetdata/dictionary/dictionary.c | 6 +- libnetdata/dictionary/dictionary.h | 2 +- libnetdata/eval/eval.c | 4 +- libnetdata/json/json.c | 2 +- libnetdata/libnetdata.h | 2 + libnetdata/onewayalloc/onewayalloc.c | 13 ++ libnetdata/onewayalloc/onewayalloc.h | 2 + libnetdata/string/string.c | 35 ++- libnetdata/worker_utilization/worker_utilization.c | 19 +- libnetdata/worker_utilization/worker_utilization.h | 1 + ml/Query.h | 8 +- parser/parser.c | 2 +- streaming/receiver.c | 2 + streaming/replication.c | 64 ++++- streaming/replication.h | 3 + streaming/rrdpush.c | 15 +- streaming/rrdpush.h | 13 +- streaming/sender.c | 104 +++++---- web/api/badges/web_buffer_svg.c | 2 +- web/api/health/health_cmdapi.c | 2 +- web/api/queries/query.c | 8 +- web/api/web_api_v1.c | 8 +- web/server/web_client_cache.c | 8 +- web/server/web_server.c | 2 +- 83 files changed, 923 insertions(+), 445 deletions(-) diff --git a/aclk/aclk.c b/aclk/aclk.c index daa4a17642..1a6ebe5509 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -947,7 +947,7 @@ char *aclk_state(void) #ifndef ENABLE_ACLK return strdupz("ACLK Available: No"); #else - BUFFER *wb = buffer_create(1024); + BUFFER *wb = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk); struct tm *tmptr, tmbuf; char *ret; diff --git a/aclk/aclk_otp.c b/aclk/aclk_otp.c index 940d5bdbe5..391313ffe7 100644 --- a/aclk/aclk_otp.c +++ b/aclk/aclk_otp.c @@ -314,7 +314,7 @@ int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char ** https_req_t req = HTTPS_REQ_T_INITIALIZER; https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER; - BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20); + BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk); req.host = target->host; req.port = target->port; @@ -394,8 +394,8 @@ int aclk_send_otp_response(const char *agent_id, const unsigned char *response, base64_encode_helper(base64, &len, response, response_bytes); - BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20); - BUFFER *resp_json = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20); + BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk); + BUFFER *resp_json = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk); buffer_sprintf(url, "%s/node/%s/password", target->path, agent_id); buffer_sprintf(resp_json, "{\"response\":\"%s\"}", base64); @@ -814,7 +814,7 @@ exit: } int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port) { - BUFFER *buf = buffer_create(1024); + BUFFER *buf = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk); https_req_t req = HTTPS_REQ_T_INITIALIZER; https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER; diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 3abe21b090..9eced08111 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -62,19 +62,19 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) int retval = 0; usec_t t; BUFFER *local_buffer = NULL; - BUFFER *log_buffer = buffer_create(NETDATA_WEB_REQUEST_URL_SIZE); + BUFFER *log_buffer = buffer_create(NETDATA_WEB_REQUEST_URL_SIZE, &netdata_buffers_statistics.buffers_aclk); RRDHOST *query_host = localhost; #ifdef NETDATA_WITH_ZLIB int z_ret; - BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); + BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk); char *start, *end; #endif struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client)); - w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); - w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE); - w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE); + w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk); + w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk); + w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk); strcpy(w->origin, "*"); // Simulate web_client_create_on_fd() w->cookie1[0] = 0; // Simulate web_client_create_on_fd() w->cookie2[0] = 0; // Simulate web_client_create_on_fd() @@ -191,7 +191,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) w->response.data->date = w->tv_ready.tv_sec; web_client_build_http_header(w); - local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE); + local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk); local_buffer->contenttype = CT_APPLICATION_JSON; buffer_strcat(local_buffer, w->response.header_output->buffer); diff --git a/aclk/https_client.c b/aclk/https_client.c index 8af85f8546..e2a42eef38 100644 --- a/aclk/https_client.c +++ b/aclk/https_client.c @@ -8,6 +8,8 @@ #include "aclk_util.h" +#include "daemon/global_statistics.h" + enum http_parse_state { HTTP_PARSE_INITIAL = 0, HTTP_PARSE_HEADERS, @@ -354,7 +356,7 @@ static int read_parse_response(https_req_ctx_t *ctx) { #define TX_BUFFER_SIZE 8192 #define RX_BUFFER_SIZE (TX_BUFFER_SIZE*2) static int handle_http_request(https_req_ctx_t *ctx) { - BUFFER *hdr = buffer_create(TX_BUFFER_SIZE); + BUFFER *hdr = buffer_create(TX_BUFFER_SIZE, &netdata_buffers_statistics.buffers_aclk); int rc = 0; http_parse_ctx_clear(&ctx->parse_ctx); diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c index 8cc39c8dbe..565d99c48b 100644 --- a/collectors/apps.plugin/apps_plugin.c +++ b/collectors/apps.plugin/apps_plugin.c @@ -4396,7 +4396,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi unsigned int memory_divisor = 1024; unsigned int io_divisor = 1024 * RATES_DETAIL; - BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX); + BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); buffer_sprintf(wb, "{" "\n \"status\":%d" diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index 915c63c4b7..394cfb93c5 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -319,7 +319,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { , SIMPLE_PATTERN_EXACT ); - dict_mountpoints = dictionary_create(DICT_OPTION_NONE); + dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors); } struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point); diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 264d0eca41..50bdc0b301 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -530,7 +530,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may } void inflight_functions_init(PARSER *parser) { - parser->inflight.functions = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions); dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser); dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser); dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser); diff --git a/collectors/proc.plugin/proc_self_mountinfo.c b/collectors/proc.plugin/proc_self_mountinfo.c index 9310f2ffc1..71c6b02994 100644 --- a/collectors/proc.plugin/proc_self_mountinfo.c +++ b/collectors/proc.plugin/proc_self_mountinfo.c @@ -227,8 +227,9 @@ struct mountinfo *mountinfo_read(int do_statvfs) { struct mountinfo *root = NULL, *last = NULL, *mi = NULL; // create a dictionary to track uniqueness - DICTIONARY *dict = dictionary_create( - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE); + DICTIONARY *dict = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE, + &dictionary_stats_category_collectors); unsigned long l, lines = procfile_lines(ff); for(l = 0; l < lines ;l++) { diff --git a/collectors/proc.plugin/proc_spl_kstat_zfs.c b/collectors/proc.plugin/proc_spl_kstat_zfs.c index 58c84050a7..610379ed9d 100644 --- a/collectors/proc.plugin/proc_spl_kstat_zfs.c +++ b/collectors/proc.plugin/proc_spl_kstat_zfs.c @@ -322,7 +322,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt) snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs"); dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename); - zfs_pools = dictionary_create(DICT_OPTION_SINGLE_THREADED); + zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors); do_zfs_pool_state = 1; } diff --git a/collectors/proc.plugin/sys_block_zram.c b/collectors/proc.plugin/sys_block_zram.c index 6bae542438..4ea015569b 100644 --- a/collectors/proc.plugin/sys_block_zram.c +++ b/collectors/proc.plugin/sys_block_zram.c @@ -267,7 +267,7 @@ int do_sys_block_zram(int update_every, usec_t dt) { } procfile_close(ff); - devices = dictionary_create(DICT_OPTION_SINGLE_THREADED); + devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors); device_count = init_devices(devices, (unsigned int)zram_id, update_every); } diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c index 7a28461a99..e6efdafda2 100644 --- a/collectors/statsd.plugin/statsd.c +++ b/collectors/statsd.plugin/statsd.c @@ -595,7 +595,7 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) { } if (unlikely(!m->set.dict)) { - m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); + m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m); m->set.unique = 0; } @@ -635,7 +635,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value statsd_reset_metric(m); if (unlikely(!m->dictionary.dict)) { - m->dictionary.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); + m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m); m->dictionary.unique = 0; } @@ -1337,7 +1337,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA else if(app) { if(!strcmp(s, "dictionary")) { if(!app->dict) - app->dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); + app->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors); dict = app->dict; } @@ -2422,13 +2422,13 @@ void *statsd_main(void *ptr) { netdata_thread_cleanup_push(statsd_main_cleanup, ptr); - statsd.gauges.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.meters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.counters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.histograms.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.dictionaries.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.sets.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); - statsd.timers.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS); + statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.histograms.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.dictionaries.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.sets.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); + statsd.timers.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors); dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges); dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters); diff --git a/collectors/tc.plugin/plugin_tc.c b/collectors/tc.plugin/plugin_tc.c index 331a787f34..72d86a0ea0 100644 --- a/collectors/tc.plugin/plugin_tc.c +++ b/collectors/tc.plugin/plugin_tc.c @@ -98,7 +98,7 @@ static bool tc_class_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse static void tc_class_index_init(struct tc_device *d) { if(!d->classes) { - d->classes = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED); + d->classes = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors); dictionary_register_delete_callback(d->classes, tc_class_free_callback, d); dictionary_register_conflict_callback(d->classes, tc_class_conflict_callback, d); @@ -144,8 +144,9 @@ static void tc_device_free_callback(const DICTIONARY_ITEM *item __maybe_unused, static void tc_device_index_init() { if(!tc_device_root_index) { - tc_device_root_index = dictionary_create( - DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT); + tc_device_root_index = dictionary_create_advanced( + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT, + &dictionary_stats_category_collectors); dictionary_register_insert_callback(tc_device_root_index, tc_device_add_callback, NULL); dictionary_register_delete_callback(tc_device_root_index, tc_device_free_callback, NULL); diff --git a/daemon/analytics.c b/daemon/analytics.c index dd7eeebbd7..a2f52bc8f6 100644 --- a/daemon/analytics.c +++ b/daemon/analytics.c @@ -241,7 +241,7 @@ void analytics_exporters(void) { //when no exporters are available, an empty string will be sent //decide if something else is more suitable (but probably not null) - BUFFER *bi = buffer_create(1000); + BUFFER *bi = buffer_create(1000, NULL); analytics_exporting_connectors(bi); analytics_set_data_str(&analytics_data.netdata_exporting_connectors, (char *)buffer_tostring(bi)); buffer_free(bi); @@ -278,7 +278,7 @@ void analytics_collectors(void) RRDSET *st; DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); char name[500]; - BUFFER *bt = buffer_create(1000); + BUFFER *bt = buffer_create(1000, NULL); rrdset_foreach_read(st, localhost) { if(!rrdset_is_available_for_viewers(st)) @@ -333,7 +333,7 @@ void analytics_alarms_notifications(void) debug(D_ANALYTICS, "Executing %s", script); - BUFFER *b = buffer_create(1000); + BUFFER *b = buffer_create(1000, NULL); int cnt = 0; FILE *fp_child_input; FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input); @@ -380,7 +380,7 @@ void analytics_get_install_type(void) */ void analytics_https(void) { - BUFFER *b = buffer_create(30); + BUFFER *b = buffer_create(30, NULL); #ifdef ENABLE_HTTPS analytics_exporting_connectors_ssl(b); buffer_strcat(b, netdata_ssl_client_ctx && rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) && localhost->sender->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ? "streaming|" : "|"); @@ -675,7 +675,7 @@ void set_late_global_environment() analytics_set_data_str(&analytics_data.netdata_config_release_channel, (char *)get_release_channel()); { - BUFFER *bi = buffer_create(1000); + BUFFER *bi = buffer_create(1000, NULL); analytics_build_info(bi); analytics_set_data_str(&analytics_data.netdata_buildinfo, (char *)buffer_tostring(bi)); buffer_free(bi); @@ -836,7 +836,7 @@ void set_global_environment() setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1); { - BUFFER *user_plugins_dirs = buffer_create(FILENAME_MAX); + BUFFER *user_plugins_dirs = buffer_create(FILENAME_MAX, NULL); for (size_t i = 1; i < PLUGINSD_MAX_DIRECTORIES && plugin_directories[i]; i++) { if (i > 1) diff --git a/daemon/commands.c b/daemon/commands.c index 8f09e63e11..377a4002f3 100644 --- a/daemon/commands.c +++ b/daemon/commands.c @@ -220,7 +220,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message) info("COMMAND: reloading host labels."); reload_host_labels(); - BUFFER *wb = buffer_create(10); + BUFFER *wb = buffer_create(10, NULL); rrdlabels_log_to_buffer(localhost->rrdlabels, wb); (*message)=strdupz(buffer_tostring(wb)); buffer_free(wb); diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 3af4368c48..2422b5a3a4 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -20,6 +20,11 @@ bool global_statistics_enabled = true; +struct netdata_buffers_statistics netdata_buffers_statistics = {}; + +static size_t dbengine_total_memory = 0; +size_t rrddim_db_memory_size = 0; + static struct global_statistics { uint16_t connected_clients; @@ -214,6 +219,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t } } +#define dictionary_stats_memory_total(stats) \ + ((stats).memory.dict + (stats).memory.values + (stats).memory.index) + static void global_statistics_charts(void) { static unsigned long long old_web_requests = 0, old_web_usec = 0, @@ -270,23 +278,168 @@ static void global_statistics_charts(void) { // ---------------------------------------------------------------- { - static RRDSET *st_uptime = NULL; - static RRDDIM *rd_uptime = NULL; + static RRDSET *st_memory = NULL; + static RRDDIM *rd_database = NULL; + static RRDDIM *rd_collectors = NULL; + static RRDDIM *rd_hosts = NULL; + static RRDDIM *rd_rrd = NULL; + static RRDDIM *rd_contexts = NULL; + static RRDDIM *rd_health = NULL; + static RRDDIM *rd_functions = NULL; + static RRDDIM *rd_labels = NULL; + static RRDDIM *rd_strings = NULL; + static RRDDIM *rd_streaming = NULL; + static RRDDIM *rd_replication = NULL; + static RRDDIM *rd_buffers = NULL; + static RRDDIM *rd_workers = NULL; + static RRDDIM *rd_other = NULL; + + if (unlikely(!st_memory)) { + st_memory = rrdset_create_localhost( + "netdata", + "memory", + NULL, + "netdata", + NULL, + "Netdata Memory", + "bytes", + "netdata", + "stats", + 130100, + localhost->rrd_update_every, + RRDSET_TYPE_STACKED); - if (unlikely(!st_uptime)) { - st_uptime = rrdset_create_localhost( + rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_hosts = rrddim_add(st_memory, "hosts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_rrd = rrddim_add(st_memory, "rrdset rrddim", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_contexts = rrddim_add(st_memory, "contexts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_health = rrddim_add(st_memory, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_functions = rrddim_add(st_memory, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_labels = rrddim_add(st_memory, "labels", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_strings = rrddim_add(st_memory, "strings", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_streaming = rrddim_add(st_memory, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + size_t buffers = + netdata_buffers_statistics.query_targets_size + + netdata_buffers_statistics.rrdset_done_rda_size + + netdata_buffers_statistics.buffers_aclk + + netdata_buffers_statistics.buffers_api + + netdata_buffers_statistics.buffers_functions + + netdata_buffers_statistics.buffers_sqlite + + netdata_buffers_statistics.buffers_exporters + + netdata_buffers_statistics.buffers_health + + netdata_buffers_statistics.buffers_streaming + + netdata_buffers_statistics.cbuffers_streaming + + netdata_buffers_statistics.buffers_web + + replication_allocated_buffers(); + + size_t strings = 0; + string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL); + + rrddim_set_by_pointer(st_memory, rd_database, (collected_number)dbengine_total_memory + (collected_number)rrddim_db_memory_size); + rrddim_set_by_pointer(st_memory, rd_collectors, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_collectors)); + rrddim_set_by_pointer(st_memory, rd_hosts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhost) + (collected_number)netdata_buffers_statistics.rrdhost_allocations_size); + rrddim_set_by_pointer(st_memory, rd_rrd, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdset_rrddim)); + rrddim_set_by_pointer(st_memory, rd_contexts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdcontext)); + rrddim_set_by_pointer(st_memory, rd_health, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhealth)); + rrddim_set_by_pointer(st_memory, rd_functions, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions)); + rrddim_set_by_pointer(st_memory, rd_labels, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdlabels)); + rrddim_set_by_pointer(st_memory, rd_strings, (collected_number)strings); + rrddim_set_by_pointer(st_memory, rd_streaming, (collected_number)netdata_buffers_statistics.rrdhost_senders + (collected_number)netdata_buffers_statistics.rrdhost_receivers); + rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory()); + rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers); + rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory()); + rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other)); + + rrdset_done(st_memory); + } + + { + static RRDSET *st_memory_buffers = NULL; + static RRDDIM *rd_queries = NULL; + static RRDDIM *rd_collectors = NULL; + static RRDDIM *rd_buffers_aclk = NULL; + static RRDDIM *rd_buffers_api = NULL; + static RRDDIM *rd_buffers_functions = NULL; + static RRDDIM *rd_buffers_sqlite = NULL; + static RRDDIM *rd_buffers_exporters = NULL; + static RRDDIM *rd_buffers_health = NULL; + static RRDDIM *rd_buffers_streaming = NULL; + static RRDDIM *rd_cbuffers_streaming = NULL; + static RRDDIM *rd_buffers_replication = NULL; + static RRDDIM *rd_buffers_web = NULL; + + if (unlikely(!st_memory_buffers)) { + st_memory_buffers = rrdset_create_localhost( "netdata", - "uptime", + "memory_buffers", NULL, "netdata", NULL, - "Netdata uptime", - "seconds", + "Netdata Memory Buffers", + "bytes", "netdata", "stats", - 130100, + 130101, localhost->rrd_update_every, - RRDSET_TYPE_LINE); + RRDSET_TYPE_STACKED); + + rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_collectors = rrddim_add(st_memory_buffers, "collection", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_aclk = rrddim_add(st_memory_buffers, "aclk", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_api = rrddim_add(st_memory_buffers, "api", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_functions = rrddim_add(st_memory_buffers, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_sqlite = rrddim_add(st_memory_buffers, "sqlite", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_exporters = rrddim_add(st_memory_buffers, "exporters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_health = rrddim_add(st_memory_buffers, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_streaming = rrddim_add(st_memory_buffers, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory()); + rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_api, (collected_number)netdata_buffers_statistics.buffers_api); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_functions, (collected_number)netdata_buffers_statistics.buffers_functions); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_sqlite, (collected_number)netdata_buffers_statistics.buffers_sqlite); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_exporters, (collected_number)netdata_buffers_statistics.buffers_exporters); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming); + rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers()); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web); + + rrdset_done(st_memory_buffers); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_uptime = NULL; + static RRDDIM *rd_uptime = NULL; + + if (unlikely(!st_uptime)) { + st_uptime = rrdset_create_localhost( + "netdata", + "uptime", + NULL, + "netdata", + NULL, + "Netdata uptime", + "seconds", + "netdata", + "stats", + 130150, + localhost->rrd_update_every, + RRDSET_TYPE_LINE); rd_uptime = rrddim_add(st_uptime, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -1076,13 +1229,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_cache_hit_ratio)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_hit_ratio", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Hit Ratio", name); ptrs->st_cache_hit_ratio = rrdset_create_localhost( @@ -1124,13 +1277,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_operations)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_operations", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Operations", name); ptrs->st_operations = rrdset_create_localhost( @@ -1178,13 +1331,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_memory)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_memory", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Memory", name); ptrs->st_pgc_memory = rrdset_create_localhost( @@ -1232,13 +1385,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_tm)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_target_memory", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Target Cache Memory", name); ptrs->st_pgc_tm = rrdset_create_localhost( @@ -1282,13 +1435,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_pages)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_pages", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Pages", name); ptrs->st_pgc_pages = rrdset_create_localhost( @@ -1326,13 +1479,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_memory_changes)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_memory_changes", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Memory Changes", name); ptrs->st_pgc_memory_changes = rrdset_create_localhost( @@ -1368,13 +1521,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_memory_migrations)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_memory_migrations", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Memory Migrations", name); ptrs->st_pgc_memory_migrations = rrdset_create_localhost( @@ -1408,13 +1561,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_memory_events)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_events", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Events", name); ptrs->st_pgc_memory_events = rrdset_create_localhost( @@ -1450,13 +1603,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_waste)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_waste_events", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Waste Events", name); ptrs->st_pgc_waste = rrdset_create_localhost( @@ -1502,13 +1655,13 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p { if (unlikely(!ptrs->st_pgc_workers)) { - BUFFER *id = buffer_create(100); + BUFFER *id = buffer_create(100, NULL); buffer_sprintf(id, "dbengine_%s_cache_workers", name); - BUFFER *family = buffer_create(100); + BUFFER *family = buffer_create(100, NULL); buffer_sprintf(family, "dbengine %s cache", name); - BUFFER *title = buffer_create(100); + BUFFER *title = buffer_create(100, NULL); buffer_sprintf(title, "Netdata %s Cache Workers", name); ptrs->st_pgc_workers = rrdset_create_localhost( @@ -1587,6 +1740,8 @@ static void dbengine2_statistics_charts(void) { buffers_total_size += buffers.julyl; #endif + dbengine_total_memory = pgc_main_stats.size + pgc_open_stats.size + pgc_extent_stats.size + mrg_stats.size + buffers_total_size; + size_t priority = 135000; { @@ -1620,6 +1775,7 @@ static void dbengine2_statistics_charts(void) { } priority++; + rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_main, (collected_number)pgc_main_stats.size); rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_open, (collected_number)pgc_open_stats.size); rrddim_set_by_pointer(st_pgc_memory, rd_pgc_memory_extent, (collected_number)pgc_extent_stats.size); @@ -2549,6 +2705,15 @@ static void update_heartbeat_charts() { // --------------------------------------------------------------------------------------------------------------------- // dictionary statistics +struct dictionary_stats dictionary_stats_category_collectors = { .name = "collectors" }; +struct dictionary_stats dictionary_stats_category_rrdhost = { .name = "rrdhost" }; +struct dictionary_stats dictionary_stats_category_rrdset_rrddim = { .name = "rrdset_rrddim" }; +struct dictionary_stats dictionary_stats_category_rrdcontext = { .name = "context" }; +struct dictionary_stats dictionary_stats_category_rrdlabels = { .name = "labels" }; +struct dictionary_stats dictionary_stats_category_rrdhealth = { .name = "health" }; +struct dictionary_stats dictionary_stats_category_functions = { .name = "functions" }; +struct dictionary_stats dictionary_stats_category_replication = { .name = "replication" }; + struct dictionary_categories { struct dictionary_stats *stats; const char *family; @@ -2594,7 +2759,15 @@ struct dictionary_categories { RRDDIM *rd_spins_delete; } dictionary_categories[] = { - { .stats = &dictionary_stats_category_other, "dictionaries", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_collectors, "dictionaries collectors", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_rrdhost, "dictionaries hosts", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_rrdset_rrddim, "dictionaries rrd", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_rrdcontext, "dictionaries contexts", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_rrdlabels, "dictionaries labels", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_rrdhealth, "dictionaries health", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_functions, "dictionaries functions", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_replication, "dictionaries replication", "dictionaries", 900000 }, + { .stats = &dictionary_stats_category_other, "dictionaries other", "dictionaries", 900000 }, // terminator { .stats = NULL, NULL, NULL, 0 }, @@ -2804,7 +2977,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) { // ------------------------------------------------------------------------ total = 0; - load_dictionary_stats_entry(memory.indexed); + load_dictionary_stats_entry(memory.index); load_dictionary_stats_entry(memory.values); load_dictionary_stats_entry(memory.dict); @@ -2838,7 +3011,7 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) { rrdlabels_add(c->st_memory->rrdlabels, "category", stats.name, RRDLABEL_SRC_AUTO); } - rrddim_set_by_pointer(c->st_memory, c->rd_memory_indexed, (collected_number)stats.memory.indexed); + rrddim_set_by_pointer(c->st_memory, c->rd_memory_indexed, (collected_number)stats.memory.index); rrddim_set_by_pointer(c->st_memory, c->rd_memory_values, (collected_number)stats.memory.values); rrddim_set_by_pointer(c->st_memory, c->rd_memory_dict, (collected_number)stats.memory.dict); diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h index c7f53781aa..7bdb153dd9 100644 --- a/daemon/global_statistics.h +++ b/daemon/global_statistics.h @@ -5,6 +5,34 @@ #include "database/rrd.h" +extern struct netdata_buffers_statistics { + size_t rrdhost_allocations_size; + size_t rrdhost_senders; + size_t rrdhost_receivers; + size_t query_targets_size; + size_t rrdset_done_rda_size; + size_t buffers_aclk; + size_t buffers_api; + size_t buffers_functions; + size_t buffers_sqlite; + size_t buffers_exporters; + size_t buffers_health; + size_t buffers_streaming; + size_t cbuffers_streaming; + size_t buffers_web; +} netdata_buffers_statistics; + +extern struct dictionary_stats dictionary_stats_category_collectors; +extern struct dictionary_stats dictionary_stats_category_rrdhost; +extern struct dictionary_stats dictionary_stats_category_rrdset_rrddim; +extern struct dictionary_stats dictionary_stats_category_rrdcontext; +extern struct dictionary_stats dictionary_stats_category_rrdlabels; +extern struct dictionary_stats dictionary_stats_category_rrdhealth; +extern struct dictionary_stats dictionary_stats_category_functions; +extern struct dictionary_stats dictionary_stats_category_replication; + +extern size_t rrddim_db_memory_size; + // ---------------------------------------------------------------------------- // global statistics diff --git a/daemon/main.c b/daemon/main.c index 408b7a2ded..2c10cb00bb 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -174,8 +174,8 @@ static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) { } static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { - BUFFER *service_list = buffer_create(1024); - BUFFER *thread_list = buffer_create(1024); + BUFFER *service_list = buffer_create(1024, NULL); + BUFFER *thread_list = buffer_create(1024, NULL); usec_t started_ut = now_monotonic_usec(), ended_ut; size_t running; SERVICE_TYPE running_services = 0; @@ -1926,6 +1926,7 @@ int main(int argc, char **argv) { netdata_anonymous_statistics_enabled=-1; struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info)); + __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED); get_system_info(system_info); system_info->hops = 0; get_install_type(&system_info->install_type, &system_info->prebuilt_arch, &system_info->prebuilt_dist); diff --git a/daemon/service.c b/daemon/service.c index 3f5e8c55a3..9761abd02d 100644 --- a/daemon/service.c +++ b/daemon/service.c @@ -52,13 +52,13 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) { size_t tiers_available = 0, tiers_said_no_retention = 0; for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(rd->tiers[tier]) { + if(rd->tiers[tier].db_collection_handle) { tiers_available++; - if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle)) + if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle)) tiers_said_no_retention++; - rd->tiers[tier]->db_collection_handle = NULL; + rd->tiers[tier].db_collection_handle = NULL; } } diff --git a/daemon/unit_test.c b/daemon/unit_test.c index efb2d3cd9d..9845ed430f 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -439,7 +439,7 @@ int unit_test_str2ld() { } int unit_test_buffer() { - BUFFER *wb = buffer_create(1); + BUFFER *wb = buffer_create(1, NULL); char string[2048 + 1]; char final[9000 + 1]; int i; @@ -1349,7 +1349,7 @@ static int test_variable_renames(void) { rrddim_reset_name(st, rd2, "DIM2NAME2"); fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2)); - BUFFER *buf = buffer_create(1); + BUFFER *buf = buffer_create(1, NULL); health_api_v1_chart_variables2json(st, buf); fprintf(stderr, "%s", buffer_tostring(buf)); buffer_free(buf); @@ -1604,7 +1604,7 @@ int test_sqlite(void) { return 1; } - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE); + BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, NULL); char *uuid_str = "0000_000"; buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str); @@ -1861,7 +1861,7 @@ static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDI // Fluh pages for subsequent real values for (i = 0 ; i < CHARTS ; ++i) { for (j = 0; j < DIMS; ++j) { - rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle); + rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle); } } } @@ -1880,7 +1880,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS // feed it with the test data for (i = 0 ; i < CHARTS ; ++i) { for (j = 0 ; j < DIMS ; ++j) { - rd[i][j]->tiers[0]->collect_ops->change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every); + rd[i][j]->tiers[0].collect_ops->change_collection_frequency(rd[i][j]->tiers[0].db_collection_handle, update_every); rd[i][j]->last_collected_time.tv_sec = st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now; @@ -1931,13 +1931,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI time_now = time_start + (c + 1) * update_every; for (i = 0 ; i < CHARTS ; ++i) { for (j = 0; j < DIMS; ++j) { - rd[i][j]->tiers[0]->query_ops->init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL); + rd[i][j]->tiers[0].query_ops->init(rd[i][j]->tiers[0].db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, STORAGE_PRIORITY_NORMAL); for (k = 0; k < QUERY_BATCH; ++k) { last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + k; expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS)); - STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops->next_metric(&handle); + STORAGE_POINT sp = rd[i][j]->tiers[0].query_ops->next_metric(&handle); value = sp.sum; time_retrieved = sp.start_time_s; end_time = sp.end_time_s; @@ -1959,7 +1959,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI errors++; } } - rd[i][j]->tiers[0]->query_ops->finalize(&handle); + rd[i][j]->tiers[0].query_ops->finalize(&handle); } } } @@ -2084,7 +2084,7 @@ int test_dbengine(void) for (i = 0 ; i < CHARTS ; ++i) { st[i]->update_every = update_every; for (j = 0; j < DIMS; ++j) { - rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle); + rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle); } } @@ -2103,7 +2103,7 @@ int test_dbengine(void) for (i = 0 ; i < CHARTS ; ++i) { st[i]->update_every = update_every; for (j = 0; j < DIMS; ++j) { - rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0]->db_collection_handle); + rrdeng_store_metric_flush_current_page((rd[i][j])->tiers[0].db_collection_handle); } } @@ -2270,7 +2270,7 @@ static void generate_dbengine_chart(void *arg) thread_info->time_max = time_current; } for (j = 0; j < DSET_DIMS; ++j) { - rrdeng_store_metric_finalize((rd[j])->tiers[0]->db_collection_handle); + rrdeng_store_metric_finalize((rd[j])->tiers[0].db_collection_handle); } } @@ -2390,13 +2390,13 @@ static void query_dbengine_chart(void *arg) time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */ } - rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL); + rd->tiers[0].query_ops->init(rd->tiers[0].db_metric_handle, &handle, time_after, time_before, STORAGE_PRIORITY_NORMAL); ++thread_info->queries_nr; for (time_now = time_after ; time_now <= time_before ; time_now += update_every) { generatedv = generate_dbengine_chart_value(i, j, time_now); expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS)); - if (unlikely(rd->tiers[0]->query_ops->is_finished(&handle))) { + if (unlikely(rd->tiers[0].query_ops->is_finished(&handle))) { if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT ", found data gap, ### E R R O R ###\n", @@ -2406,7 +2406,7 @@ static void query_dbengine_chart(void *arg) break; } - STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle); + STORAGE_POINT sp = rd->tiers[0].query_ops->next_metric(&handle); value = sp.sum; time_retrieved = sp.start_time_s; end_time = sp.end_time_s; @@ -2444,7 +2444,7 @@ static void query_dbengine_chart(void *arg) } } } - rd->tiers[0]->query_ops->finalize(&handle); + rd->tiers[0].query_ops->finalize(&handle); } while(!thread_info->done); if(value_errors) diff --git a/database/engine/cache.c b/database/engine/cache.c index 8d462edee9..7993535b3f 100644 --- a/database/engine/cache.c +++ b/database/engine/cache.c @@ -92,6 +92,7 @@ struct pgc { size_t additional_bytes_per_page; free_clean_page_callback pgc_free_clean_cb; save_dirty_page_callback pgc_save_dirty_cb; + save_dirty_init_callback pgc_save_init_cb; PGC_OPTIONS options; size_t severe_pressure_per1000; @@ -205,7 +206,7 @@ static inline void pointer_del(PGC *cache __maybe_unused, PGC_PAGE *page __maybe // ---------------------------------------------------------------------------- // locking -static size_t pgc_indexing_partition(PGC *cache, Word_t metric_id) { +static inline size_t pgc_indexing_partition(PGC *cache, Word_t metric_id) { static __thread Word_t last_metric_id = 0; static __thread size_t last_partition = 0; @@ -218,19 +219,19 @@ static size_t pgc_indexing_partition(PGC *cache, Word_t metric_id) { return last_partition; } -static void pgc_index_read_lock(PGC *cache, size_t partition) { +static inline void pgc_index_read_lock(PGC *cache, size_t partition) { netdata_rwlock_rdlock(&cache->index[partition].rwlock); } -static void pgc_index_read_unlock(PGC *cache, size_t partition) { +static inline void pgc_index_read_unlock(PGC *cache, size_t partition) { netdata_rwlock_unlock(&cache->index[partition].rwlock); } -//static bool pgc_index_write_trylock(PGC *cache, size_t partition) { +//static inline bool pgc_index_write_trylock(PGC *cache, size_t partition) { // return !netdata_rwlock_trywrlock(&cache->index[partition].rwlock); //} -static void pgc_index_write_lock(PGC *cache, size_t partition) { +static inline void pgc_index_write_lock(PGC *cache, size_t partition) { netdata_rwlock_wrlock(&cache->index[partition].rwlock); } -static void pgc_index_write_unlock(PGC *cache, size_t partition) { +static inline void pgc_index_write_unlock(PGC *cache, size_t partition) { netdata_rwlock_unlock(&cache->index[partition].rwlock); } @@ -382,11 +383,11 @@ static inline bool flushing_critical(PGC *cache) { // ---------------------------------------------------------------------------- // helpers -static size_t page_assumed_size(PGC *cache, size_t size) { +static inline size_t page_assumed_size(PGC *cache, size_t size) { return size + (sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page + sizeof(Word_t) * 3); } -static size_t page_size_from_assumed_size(PGC *cache, size_t assumed_size) { +static inline size_t page_size_from_assumed_size(PGC *cache, size_t assumed_size) { return assumed_size - (sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page + sizeof(Word_t) * 3); } @@ -422,7 +423,7 @@ static ARAL section_pages_aral = { .requested_element_size = sizeof(struct section_pages), }; -static void pgc_stats_ll_judy_change(PGC *cache, struct pgc_linked_list *ll, size_t mem_before_judyl, size_t mem_after_judyl) { +static inline void pgc_stats_ll_judy_change(PGC *cache, struct pgc_linked_list *ll, size_t mem_before_judyl, size_t mem_after_judyl) { if(mem_after_judyl > mem_before_judyl) { __atomic_add_fetch(&ll->stats->size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED); __atomic_add_fetch(&cache->stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED); @@ -433,7 +434,7 @@ static void pgc_stats_ll_judy_change(PGC *cache, struct pgc_linked_list *ll, siz } } -static void pgc_stats_index_judy_change(PGC *cache, size_t mem_before_judyl, size_t mem_after_judyl) { +static inline void pgc_stats_index_judy_change(PGC *cache, size_t mem_before_judyl, size_t mem_after_judyl) { if(mem_after_judyl > mem_before_judyl) { __atomic_add_fetch(&cache->stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED); } @@ -556,7 +557,7 @@ static void pgc_ll_del(PGC *cache __maybe_unused, struct pgc_linked_list *ll, PG pgc_ll_unlock(cache, ll); } -static void page_has_been_accessed(PGC *cache, PGC_PAGE *page) { +static inline void page_has_been_accessed(PGC *cache, PGC_PAGE *page) { PGC_PAGE_FLAGS flags = page_flag_check(page, PGC_PAGE_CLEAN | PGC_PAGE_HAS_NO_DATA_IGNORE_ACCESSES); if (!(flags & PGC_PAGE_HAS_NO_DATA_IGNORE_ACCESSES)) { @@ -604,7 +605,7 @@ static inline void page_set_clean(PGC *cache, PGC_PAGE *page, bool having_transi page_transition_unlock(cache, page); } -static void page_set_dirty(PGC *cache, PGC_PAGE *page, bool having_hot_lock) { +static inline void page_set_dirty(PGC *cache, PGC_PAGE *page, bool having_hot_lock) { if(!having_hot_lock) // to avoid deadlocks, we have to get the hot lock before the page transition // since this is what all_hot_to_dirty() does @@ -830,7 +831,7 @@ static inline bool acquired_page_get_for_deletion_or_release_it(PGC *cache __may // ---------------------------------------------------------------------------- // Indexing -static void free_this_page(PGC *cache, PGC_PAGE *page) { +static inline void free_this_page(PGC *cache, PGC_PAGE *page) { // call the callback to free the user supplied memory cache->config.pgc_free_clean_cb(cache, (PGC_ENTRY){ .section = page->section, @@ -916,7 +917,7 @@ static void remove_this_page_from_index_unsafe(PGC *cache, PGC_PAGE *page, size_ pointer_del(cache, page); } -static void remove_and_free_page_not_in_any_queue_and_acquired_for_deletion(PGC *cache, PGC_PAGE *page) { +static inline void remove_and_free_page_not_in_any_queue_and_acquired_for_deletion(PGC *cache, PGC_PAGE *page) { size_t partition = pgc_indexing_partition(cache, page->metric_id); pgc_index_write_lock(cache, partition); remove_this_page_from_index_unsafe(cache, page, partition); @@ -924,7 +925,7 @@ static void remove_and_free_page_not_in_any_queue_and_acquired_for_deletion(PGC free_this_page(cache, page); } -static bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache, PGC_PAGE *page) { +static inline bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache, PGC_PAGE *page) { pointer_check(cache, page); page_transition_lock(cache, page); @@ -1632,6 +1633,9 @@ static bool flush_pages(PGC *cache, size_t max_flushes, Word_t section, bool wai continue; } + if(cache->config.pgc_save_init_cb) + cache->config.pgc_save_init_cb(cache, last_section); + pgc_ll_unlock(cache, &cache->dirty); have_dirty_lock = false; @@ -1704,7 +1708,9 @@ void free_all_unreferenced_clean_pages(PGC *cache) { // public API PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_cb, - size_t max_dirty_pages_per_flush, save_dirty_page_callback pgc_save_dirty_cb, + size_t max_dirty_pages_per_flush, + save_dirty_init_callback pgc_save_init_cb, + save_dirty_page_callback pgc_save_dirty_cb, size_t max_pages_per_inline_eviction, size_t max_inline_evictors, size_t max_skip_pages_per_inline_eviction, size_t max_flushes_inline, @@ -1723,7 +1729,8 @@ PGC *pgc_create(size_t clean_size_bytes, free_clean_page_callback pgc_free_cb, cache->config.options = options; cache->config.clean_size = (clean_size_bytes < 1 * 1024 * 1024) ? 1 * 1024 * 1024 : clean_size_bytes; cache->config.pgc_free_clean_cb = pgc_free_cb; - cache->config.max_dirty_pages_per_call = max_dirty_pages_per_flush, + cache->config.max_dirty_pages_per_call = max_dirty_pages_per_flush; + cache->config.pgc_save_init_cb = pgc_save_init_cb; cache->config.pgc_save_dirty_cb = pgc_save_dirty_cb; cache->config.max_pages_per_inline_eviction = (max_pages_per_inline_eviction < 2) ? 2 : max_pages_per_inline_eviction; cache->config.max_skip_pages_per_inline_eviction = (max_skip_pages_per_inline_eviction < 2) ? 2 : max_skip_pages_per_inline_eviction; @@ -2579,7 +2586,7 @@ void unittest_stress_test(void) { int pgc_unittest(void) { PGC *cache = pgc_create(32 * 1024 * 1024, unittest_free_clean_p