diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-10-09 21:58:21 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-09 21:58:21 +0300 |
commit | 758d9c405d2d768a3c125052a02c7a1503b01bd8 (patch) | |
tree | de24c46008c9c7bf95270ebb8f3b117229c43db9 | |
parent | 067305602f373d12286e492143bf6cb2a32ffe31 (diff) |
full memory tracking and profiling of Netdata Agent (#13789)
* full memory tracking and profiling of Netdata Agent
* initialize dbengine only when it is needed
* handling of dbengine compiled but not available
* restore unittest
* restore unittest again
* more improvements about ifdef dbengine
* fix compilation when dbengine is not enabled
* check if dbengine is enabled on exit
* call freez() not free()
* aral unittest
* internal checks activate trace allocations; dev mode activates internal checks
-rw-r--r-- | aclk/aclk_tx_msgs.c | 11 | ||||
-rw-r--r-- | daemon/global_statistics.c | 162 | ||||
-rw-r--r-- | daemon/main.c | 20 | ||||
-rw-r--r-- | daemon/service.c | 2 | ||||
-rw-r--r-- | database/engine/journalfile.c | 2 | ||||
-rw-r--r-- | database/engine/rrdengine.c | 2 | ||||
-rw-r--r-- | database/ram/rrddim_mem.c | 6 | ||||
-rw-r--r-- | database/rrd.h | 1 | ||||
-rw-r--r-- | database/rrdcontext.c | 6 | ||||
-rw-r--r-- | database/rrddim.c | 15 | ||||
-rw-r--r-- | database/rrdhost.c | 78 | ||||
-rw-r--r-- | database/rrdset.c | 2 | ||||
-rw-r--r-- | database/sqlite/sqlite_aclk.c | 9 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.c | 6 | ||||
-rw-r--r-- | libnetdata/arrayalloc/arrayalloc.c | 125 | ||||
-rw-r--r-- | libnetdata/arrayalloc/arrayalloc.h | 14 | ||||
-rw-r--r-- | libnetdata/config/appconfig.c | 2 | ||||
-rw-r--r-- | libnetdata/config/appconfig.h | 2 | ||||
-rw-r--r-- | libnetdata/dictionary/dictionary.c | 107 | ||||
-rw-r--r-- | libnetdata/libnetdata.c | 310 | ||||
-rw-r--r-- | libnetdata/libnetdata.h | 71 | ||||
-rw-r--r-- | libnetdata/procfile/procfile.c | 42 | ||||
-rw-r--r-- | streaming/receiver.c | 9 | ||||
-rw-r--r-- | streaming/rrdpush.c | 25 | ||||
-rw-r--r-- | streaming/rrdpush.h | 1 | ||||
-rw-r--r-- | web/api/web_api_v1.c | 7 |
26 files changed, 778 insertions, 259 deletions
diff --git a/aclk/aclk_tx_msgs.c b/aclk/aclk_tx_msgs.c index 9bf8529ef8..e58f5c3661 100644 --- a/aclk/aclk_tx_msgs.c +++ b/aclk/aclk_tx_msgs.c @@ -15,6 +15,13 @@ // version for aclk legacy (old cloud arch) #define ACLK_VERSION 2 +static void freez_aclk_publish5a(void *ptr) { + freez(ptr); +} +static void freez_aclk_publish5b(void *ptr) { + freez(ptr); +} + uint16_t aclk_send_bin_message_subtopic_pid(mqtt_wss_client client, char *msg, size_t msg_len, enum aclk_topics subtopic, const char *msgname) { #ifndef ACLK_LOG_CONVERSATION_DIR @@ -29,7 +36,7 @@ uint16_t aclk_send_bin_message_subtopic_pid(mqtt_wss_client client, char *msg, s } if (use_mqtt_5) - mqtt_wss_publish5(client, (char*)topic, NULL, msg, &freez, msg_len, MQTT_WSS_PUB_QOS1, &packet_id); + mqtt_wss_publish5(client, (char*)topic, NULL, msg, &freez_aclk_publish5a, msg_len, MQTT_WSS_PUB_QOS1, &packet_id); else mqtt_wss_publish_pid(client, topic, msg, msg_len, MQTT_WSS_PUB_QOS1, &packet_id); @@ -81,7 +88,7 @@ static int aclk_send_message_with_bin_payload(mqtt_wss_client client, json_objec } if (use_mqtt_5) - mqtt_wss_publish5(client, (char*)topic, NULL, (char*)(payload_len ? full_msg : str), (payload_len ? &freez : &json_object_put_wrapper), len, MQTT_WSS_PUB_QOS1, &packet_id); + mqtt_wss_publish5(client, (char*)topic, NULL, (char*)(payload_len ? full_msg : str), (payload_len ? &freez_aclk_publish5b : &json_object_put_wrapper), len, MQTT_WSS_PUB_QOS1, &packet_id); else { rc = mqtt_wss_publish_pid_block(client, topic, payload_len ? full_msg : str, len, MQTT_WSS_PUB_QOS1, &packet_id, 5000); freez(full_msg); diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 1d4da897a5..79e0bb7711 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -11,8 +11,9 @@ #define WORKER_JOB_HEARTBEAT 4 #define WORKER_JOB_STRINGS 5 #define WORKER_JOB_DICTIONARIES 6 +#define WORKER_JOB_MALLOC_TRACE 7 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 7 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5 #endif @@ -1571,6 +1572,153 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) { } } +#ifdef NETDATA_TRACE_ALLOCATIONS + +struct memory_trace_data { + RRDSET *st_memory; + RRDSET *st_allocations; + RRDSET *st_avg_alloc; + RRDSET *st_ops; +}; + +static int do_memory_trace_item(void *item, void *data) { + struct memory_trace_data *tmp = data; + struct malloc_trace *p = item; + + // ------------------------------------------------------------------------ + + if(!p->rd_bytes) + p->rd_bytes = rrddim_add(tmp->st_memory, p->function, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + collected_number bytes = (collected_number)__atomic_load_n(&p->bytes, __ATOMIC_RELAXED); + rrddim_set_by_pointer(tmp->st_memory, p->rd_bytes, bytes); + + // ------------------------------------------------------------------------ + + if(!p->rd_allocations) + p->rd_allocations = rrddim_add(tmp->st_allocations, p->function, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + collected_number allocs = (collected_number)__atomic_load_n(&p->allocations, __ATOMIC_RELAXED); + rrddim_set_by_pointer(tmp->st_allocations, p->rd_allocations, allocs); + + // ------------------------------------------------------------------------ + + if(!p->rd_avg_alloc) + p->rd_avg_alloc = rrddim_add(tmp->st_avg_alloc, p->function, NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); + + collected_number avg_alloc = (allocs)?(bytes * 100 / allocs):0; + rrddim_set_by_pointer(tmp->st_avg_alloc, p->rd_avg_alloc, avg_alloc); + + // ------------------------------------------------------------------------ + + if(!p->rd_ops) + p->rd_ops = rrddim_add(tmp->st_ops, p->function, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + collected_number ops = 0; + ops += (collected_number)__atomic_load_n(&p->malloc_calls, __ATOMIC_RELAXED); + ops += (collected_number)__atomic_load_n(&p->calloc_calls, __ATOMIC_RELAXED); + ops += (collected_number)__atomic_load_n(&p->realloc_calls, __ATOMIC_RELAXED); + ops += (collected_number)__atomic_load_n(&p->strdup_calls, __ATOMIC_RELAXED); + ops += (collected_number)__atomic_load_n(&p->free_calls, __ATOMIC_RELAXED); + rrddim_set_by_pointer(tmp->st_ops, p->rd_ops, ops); + + // ------------------------------------------------------------------------ + + return 1; +} +static void malloc_trace_statistics(void) { + static struct memory_trace_data tmp = { + .st_memory = NULL, + .st_allocations = NULL, + .st_avg_alloc = NULL, + .st_ops = NULL, + }; + + if(!tmp.st_memory) { + tmp.st_memory = rrdset_create_localhost( + "netdata" + , "memory_size" + , NULL + , "memory" + , "netdata.memory.size" + , "Netdata Memory Used by Function" + , "bytes" + , "netdata" + , "stats" + , 900000 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); + } + else + rrdset_next(tmp.st_memory); + + if(!tmp.st_ops) { + tmp.st_ops = rrdset_create_localhost( + "netdata" + , "memory_operations" + , NULL + , "memory" + , "netdata.memory.operations" + , "Netdata Memory Operations by Function" + , "ops/s" + , "netdata" + , "stats" + , 900001 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + } + else + rrdset_next(tmp.st_ops); + + if(!tmp.st_allocations) { + tmp.st_allocations = rrdset_create_localhost( + "netdata" + , "memory_allocations" + , NULL + , "memory" + , "netdata.memory.allocations" + , "Netdata Memory Allocations by Function" + , "allocations" + , "netdata" + , "stats" + , 900002 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); + } + else + rrdset_next(tmp.st_allocations); + + if(!tmp.st_avg_alloc) { + tmp.st_avg_alloc = rrdset_create_localhost( + "netdata" + , "memory_avg_alloc" + , NULL + , "memory" + , "netdata.memory.avg_alloc" + , "Netdata Average Allocation Size by Function" + , "bytes" + , "netdata" + , "stats" + , 900003 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + } + else + rrdset_next(tmp.st_avg_alloc); + + malloc_trace_walkthrough(do_memory_trace_item, &tmp); + + rrdset_done(tmp.st_memory); + rrdset_done(tmp.st_ops); + rrdset_done(tmp.st_allocations); + rrdset_done(tmp.st_avg_alloc); +} +#endif + static void dictionary_statistics(void) { for(int i = 0; dictionary_categories[i].stats ;i++) { update_dictionary_category_charts(&dictionary_categories[i]); @@ -2375,6 +2523,7 @@ void *global_statistics_main(void *ptr) worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine"); worker_register_job_name(WORKER_JOB_STRINGS, "strings"); worker_register_job_name(WORKER_JOB_DICTIONARIES, "dictionaries"); + worker_register_job_name(WORKER_JOB_MALLOC_TRACE, "malloc_trace"); netdata_thread_cleanup_push(global_statistics_cleanup, ptr); @@ -2404,8 +2553,10 @@ void *global_statistics_main(void *ptr) worker_is_busy(WORKER_JOB_REGISTRY); registry_statistics(); - worker_is_busy(WORKER_JOB_DBENGINE); - dbengine_statistics_charts(); + if(dbengine_enabled) { + worker_is_busy(WORKER_JOB_DBENGINE); + dbengine_statistics_charts(); + } worker_is_busy(WORKER_JOB_HEARTBEAT); update_heartbeat_charts(); @@ -2415,6 +2566,11 @@ void *global_statistics_main(void *ptr) worker_is_busy(WORKER_JOB_DICTIONARIES); dictionary_statistics(); + +#ifdef NETDATA_TRACE_ALLOCATIONS + worker_is_busy(WORKER_JOB_MALLOC_TRACE); + malloc_trace_statistics(); +#endif } netdata_thread_cleanup_pop(1); diff --git a/daemon/main.c b/daemon/main.c index 20b6068c87..8f3477ad97 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -55,13 +55,17 @@ void netdata_cleanup_and_exit(int ret) { // free the database info("EXIT: freeing database memory..."); #ifdef ENABLE_DBENGINE - for(int tier = 0; tier < storage_tiers ; tier++) - rrdeng_prepare_exit(multidb_ctx[tier]); + if(dbengine_enabled) { + for (int tier = 0; tier < storage_tiers; tier++) + rrdeng_prepare_exit(multidb_ctx[tier]); + } #endif rrdhost_free_all(); #ifdef ENABLE_DBENGINE - for(int tier = 0; tier < storage_tiers ; tier++) - rrdeng_exit(multidb_ctx[tier]); + if(dbengine_enabled) { + for (int tier = 0; tier < storage_tiers; tier++) + rrdeng_exit(multidb_ctx[tier]); + } #endif } sql_close_context_database(); @@ -255,7 +259,8 @@ void cancel_main_threads() { for (i = 0; static_threads[i].name != NULL ; i++) freez(static_threads[i].thread); - free(static_threads); + + freez(static_threads); } struct option_def option_definitions[] = { @@ -1001,6 +1006,8 @@ int main(int argc, char **argv) { if(string_unittest(10000)) return 1; if (dictionary_unittest(10000)) return 1; + if(aral_unittest(10000)) + return 1; if (rrdlabels_unittest()) return 1; if (ctx_unittest()) @@ -1023,6 +1030,9 @@ int main(int argc, char **argv) { else if(strcmp(optarg, "dicttest") == 0) { return dictionary_unittest(10000); } + else if(strcmp(optarg, "araltest") == 0) { + return aral_unittest(10000); + } else if(strcmp(optarg, "stringtest") == 0) { return string_unittest(10000); } diff --git a/daemon/service.c b/daemon/service.c index 71b377dbba..3a267402d1 100644 --- a/daemon/service.c +++ b/daemon/service.c @@ -216,10 +216,8 @@ restart_after_removal: info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid); if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) -#ifdef ENABLE_DBENGINE /* don't delete multi-host DB host files */ && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) -#endif ) { worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS); rrdhost_delete_charts(host); diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c index 0e8f935ad7..95c3a46f80 100644 --- a/database/engine/journalfile.c +++ b/database/engine/journalfile.c @@ -527,7 +527,7 @@ int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfi info("Journal file \"%s\" loaded (size:%"PRIu64").", path, file_size); if (likely(journalfile->data)) - munmap(journalfile->data, file_size); + netdata_munmap(journalfile->data, file_size); return 0; error: diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c index 97f3fe6e2f..ce808aadbc 100644 --- a/database/engine/rrdengine.c +++ b/database/engine/rrdengine.c @@ -30,7 +30,7 @@ void dbengine_page_free(void *page) { if (unlikely(db_engine_use_malloc)) freez(page); else - munmap(page, RRDENG_BLOCK_SIZE); + netdata_munmap(page, RRDENG_BLOCK_SIZE); } static void sanity_check(void) diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c index 74674dbdbd..329f8590b8 100644 --- a/database/ram/rrddim_mem.c +++ b/database/ram/rrddim_mem.c @@ -16,7 +16,7 @@ void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle) { RRDDIM *rd = (RRDDIM *)db_metric_handle; rd->db[rd->rrdset->current_entry] = pack_storage_number(NAN, SN_FLAG_NONE); - struct mem_collect_handle *ch = calloc(1, sizeof(struct mem_collect_handle)); + struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle)); ch->rd = rd; return (STORAGE_COLLECT_HANDLE *)ch; } @@ -46,7 +46,7 @@ void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) { } int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) { - free(collection_handle); + freez(collection_handle); return 0; } @@ -142,7 +142,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_qu handle->rd = rd; handle->start_time = start_time; handle->end_time = end_time; - struct mem_query_handle* h = calloc(1, sizeof(struct mem_query_handle)); + struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle)); h->slot = rrddim_time2slot(rd, start_time); h->last_slot = rrddim_time2slot(rd, end_time); h->dt = rd->rrdset->update_every; diff --git a/database/rrd.h b/database/rrd.h index f72a1f4c69..c09709148b 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -52,6 +52,7 @@ struct pg_cache_page_index; #include "sqlite/sqlite_health.h" #include "rrdcontext.h" +extern bool dbengine_enabled; extern int storage_tiers; extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS]; diff --git a/database/rrdcontext.c b/database/rrdcontext.c index 9a9d4f1582..20f20597e6 100644 --- a/database/rrdcontext.c +++ b/database/rrdcontext.c @@ -2200,7 +2200,7 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) { max_last_time_t = rrddim_last_entry_t(rm->rrddim); } #ifdef ENABLE_DBENGINE - else { + else if (dbengine_enabled) { RRDHOST *rrdhost = rm->ri->rc->rrdhost; for (int tier = 0; tier < storage_tiers; tier++) { if(!rrdhost->storage_instance[tier]) continue; @@ -2215,6 +2215,10 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) { } } } + else { + // cannot get retention + return; + } #endif if(min_first_time_t == LONG_MAX) diff --git a/database/rrddim.c b/database/rrddim.c index d5832a1dab..59ef31eed0 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -2,9 +2,6 @@ #define NETDATA_RRD_INTERNALS #include "rrd.h" -#ifdef ENABLE_DBENGINE -#include "database/engine/rrdengineapi.h" -#endif #include "storage_engine.h" // ---------------------------------------------------------------------------- @@ -27,6 +24,12 @@ struct rrddim_constructor { }; +// isolated call to appear +// separate in statistics +static void *rrddim_alloc_db(size_t entries) { + return callocz(entries, sizeof(storage_number)); +} + static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *constructor_data) { struct rrddim_constructor *ctr = constructor_data; RRDDIM *rd = rrddim; @@ -73,7 +76,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v size_t entries = st->entries; if(entries < 5) entries = 5; - rd->db = callocz(entries, sizeof(storage_number)); + rd->db = rrddim_alloc_db(entries); rd->memsize = entries * sizeof(storage_number); } @@ -222,7 +225,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v if(rd->db) { if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) - munmap(rd->db, rd->memsize); + netdata_munmap(rd->db, rd->memsize); else freez(rd->db); } @@ -641,7 +644,7 @@ void rrddim_memory_file_free(RRDDIM *rd) { struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file; freez(rd_on_file->cache_filename); - munmap(rd_on_file, rd_on_file->memsize); + netdata_munmap(rd_on_file, rd_on_file->memsize); // remove the pointers from the RRDDIM rd->rd_on_file = NULL; diff --git a/database/rrdhost.c b/database/rrdhost.c index 4b9aec96d3..de01aa948f 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -3,6 +3,7 @@ #define NETDATA_RRD_INTERNALS #include "rrd.h" +bool dbengine_enabled = false; // will become true if and when dbengine is initialized int storage_tiers = 1; int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 }; RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW }; @@ -328,12 +329,18 @@ RRDHOST *rrdhost_create(const char *hostname, ) { debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid); + rrd_check_wrlock(); + + if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) { + error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname); + memory_mode = RRD_MEMORY_MODE_ALLOC; + } + #ifdef ENABLE_DBENGINE int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid); #else - int is_legacy = 1; +int is_legacy = 1; #endif - rrd_check_wrlock(); int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy); RRDHOST *host = callocz(1, sizeof(RRDHOST)); @@ -384,8 +391,8 @@ RRDHOST *rrdhost_create(const char *hostname, host->cache_dir = strdupz(filename); } - if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || ( - host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) { + if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || + (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) { int r = mkdir(host->cache_dir, 0775); if(r != 0 && errno != EEXIST) error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir); @@ -754,22 +761,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim // ---------------------------------------------------------------------------- // RRDHOST global / startup initialization -int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { - rrdhost_init(); - - if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - fatal("Failed to initialize SQLite"); - info("Skipping SQLITE metadata initialization since memory mode is not dbengine"); - } - - if (unlikely(sql_init_context_database(system_info ? 0 : 1))) { - error_report("Failed to initialize context metadata database"); - } - - if (unlikely(!system_info)) - goto unittest; - +void dbengine_init(char *hostname) { #ifdef ENABLE_DBENGINE storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); if(storage_tiers < 1) { @@ -857,7 +849,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", hostname, tier, tier); break; } - + internal_error(true, "DBENGINE tier %d grouping iterations is set to %d", tier, storage_tiers_grouping_iterations[tier]); ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier); if(ret != 0) { @@ -877,6 +869,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { else if(!created_tiers) fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir); + dbengine_enabled = true; #else storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1); if(storage_tiers != 1) { @@ -885,10 +878,49 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); } #endif +} + +int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { + rrdhost_init(); + + if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + fatal("Failed to initialize SQLite"); + info("Skipping SQLITE metadata initialization since memory mode is not dbengine"); + } + + if (unlikely(sql_init_context_database(system_info ? 0 : 1))) { + error_report("Failed to initialize context metadata database"); + } + + if (unlikely(strcmp(hostname, "unittest") == 0)) { + dbengine_enabled = true; + goto unittest; + } - health_init(); rrdpush_init(); + if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || storage_tiers > 1 || rrdpush_receiver_needs_dbengine()) { + info("Initializing dbengine..."); + dbengine_init(hostname); + } + else + info("Not initializing dbengine..."); + + if(!dbengine_enabled) { + if (storage_tiers > 1) { + error("dbengine is not enabled, but %d tiers have been requested. Resetting tiers to 1", storage_tiers); + storage_tiers = 1; + } + + if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc"); + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; + } + } + + health_init(); + unittest: debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname); rrd_wrlock(); @@ -1418,10 +1450,8 @@ void rrdhost_cleanup_all(void) { RRDHOST *host; rrdhost_foreach_read(host) { if (host != localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver -#ifdef ENABLE_DBENGINE /* don't delete multi-host DB host files */ && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) -#endif ) rrdhost_delete_charts(host); else diff --git a/database/rrdset.c b/database/rrdset.c index 426582ca0e..9b04712afb 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -1972,7 +1972,7 @@ void rrdset_memory_file_free(RRDSET *st) { rrdset_memory_file_update(st); struct rrdset_map_save_v019 *st_on_file = st->st_on_file; - munmap(st_on_file, st_on_file->memsize); + netdata_munmap(st_on_file, st_on_file->memsize); // remove the pointers from the RRDDIM st->st_on_file = NULL; diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c index 3e34f664cb..03edd6bc8a 100644 --- a/database/sqlite/sqlite_aclk.c +++ b/database/sqlite/sqlite_aclk.c @@ -264,13 +264,6 @@ static int create_host_callback(void *data, int argc, char **argv, char **column struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info)); system_info->hops = str2i((const char *) argv[IDX_HOPS]); - RRD_MEMORY_MODE memory_mode; - -#ifdef ENABLE_DBENGINE - memory_mode = RRD_MEMORY_MODE_DBENGINE; -#else - memory_mode = RRD_MEMORY_MODE_RAM; -#endif sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info); @@ -287,7 +280,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown") , argv[3] ? str2i(argv[IDX_UPDATE_EVERY]) : 1 , argv[13] ? str2i(argv[IDX_ENTRIES]) : 0 - , memory_mode + , default_rrd_memory_mode , 0 // health , 0 // rrdpush enabled , NULL //destination diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index b7ed5db103..4137a3baa4 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -1408,8 +1408,10 @@ RRDHOST *sql_create_host_by_uuid(char *hostname) rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); #ifdef ENABLE_DBENGINE - for(int tier = 0; tier < storage_tiers ; tier++) - host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier]; + if(dbengine_enabled) { + for (int tier = 0; tier < storage_tiers; tier++) + host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier]; + } #endif failed: diff --git a/libnetdata/arrayalloc/arrayalloc.c b/libnetdata/arrayalloc/arrayalloc.c index c3e8114a83..920669d931 100644 --- a/libnetdata/arrayalloc/arrayalloc.c +++ b/libnetdata/arrayalloc/arrayalloc.c @@ -162,7 +162,11 @@ static inline ARAL_PAGE *find_page_with_allocation(ARAL *ar, void *ptr) { return page; } -static void arrayalloc_increase(ARAL *ar) { +#ifdef NETDATA_TRACE_ALLOCATIONS +static void arrayalloc_add_page(ARAL *ar, const char *file, const char *function, size_t line) { +#else +static void arrayalloc_add_page(ARAL *ar) { +#endif if(unlikely(!ar->internal.initialized)) arrayalloc_init(ar); @@ -182,8 +186,13 @@ static void arrayalloc_increase(ARAL *ar) { if (unlikely(!page->data)) fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename); } - else + else { +#ifdef NETDATA_TRACE_ALLOCATIONS + page->data = mallocz_int(page->size, file, function, line); +#else page->data = mallocz(page->size); +#endif + } // link the free space to its page ARAL_FREE *fr = (ARAL_FREE *)page->data; @@ -217,14 +226,23 @@ ARAL *arrayalloc_create(size_t element_size, size_t elements, const |