full memory tracking and profiling of Netdata Agent (#13789)

* full memory tracking and profiling of Netdata Agent

* initialize dbengine only when it is needed

* handling of dbengine compiled but not available

* restore unittest

* restore unittest again

* more improvements about ifdef dbengine

* fix compilation when dbengine is not enabled

* check if dbengine is enabled on exit

* call freez() not free()

* aral unittest

* internal checks activate trace allocations; dev mode activates internal checks

26 files changed, 778 insertions, 259 deletions

diff --git a/aclk/aclk_tx_msgs.c b/aclk/aclk_tx_msgs.c
index 9bf8529ef8..e58f5c3661 100644
--- a/aclk/aclk_tx_msgs.c
+++ b/aclk/aclk_tx_msgs.c
@@ -15,6 +15,13 @@
 // version for aclk legacy (old cloud arch)
 #define ACLK_VERSION 2
 
+static void freez_aclk_publish5a(void *ptr) {
+    freez(ptr);
+}
+static void freez_aclk_publish5b(void *ptr) {
+    freez(ptr);
+}
+
 uint16_t aclk_send_bin_message_subtopic_pid(mqtt_wss_client client, char *msg, size_t msg_len, enum aclk_topics subtopic, const char *msgname)
 {
 #ifndef ACLK_LOG_CONVERSATION_DIR
@@ -29,7 +36,7 @@ uint16_t aclk_send_bin_message_subtopic_pid(mqtt_wss_client client, char *msg, s
     }
 
     if (use_mqtt_5)
-        mqtt_wss_publish5(client, (char*)topic, NULL, msg, &freez, msg_len, MQTT_WSS_PUB_QOS1, &packet_id);
+        mqtt_wss_publish5(client, (char*)topic, NULL, msg, &freez_aclk_publish5a, msg_len, MQTT_WSS_PUB_QOS1, &packet_id);
     else
         mqtt_wss_publish_pid(client, topic, msg, msg_len,  MQTT_WSS_PUB_QOS1, &packet_id);
 
@@ -81,7 +88,7 @@ static int aclk_send_message_with_bin_payload(mqtt_wss_client client, json_objec
     }
 
     if (use_mqtt_5)
-        mqtt_wss_publish5(client, (char*)topic, NULL, (char*)(payload_len ? full_msg : str), (payload_len ? &freez : &json_object_put_wrapper), len, MQTT_WSS_PUB_QOS1, &packet_id);
+        mqtt_wss_publish5(client, (char*)topic, NULL, (char*)(payload_len ? full_msg : str), (payload_len ? &freez_aclk_publish5b : &json_object_put_wrapper), len, MQTT_WSS_PUB_QOS1, &packet_id);
     else {
         rc = mqtt_wss_publish_pid_block(client, topic, payload_len ? full_msg : str, len,  MQTT_WSS_PUB_QOS1, &packet_id, 5000);
         freez(full_msg);
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 1d4da897a5..79e0bb7711 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -11,8 +11,9 @@
 #define WORKER_JOB_HEARTBEAT          4
 #define WORKER_JOB_STRINGS            5
 #define WORKER_JOB_DICTIONARIES       6
+#define WORKER_JOB_MALLOC_TRACE       7
 
-#if WORKER_UTILIZATION_MAX_JOB_TYPES < 7
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
 #endif
 
@@ -1571,6 +1572,153 @@ static void update_dictionary_category_charts(struct dictionary_categories *c) {
     }
 }
 
+#ifdef NETDATA_TRACE_ALLOCATIONS
+
+struct memory_trace_data {
+    RRDSET *st_memory;
+    RRDSET *st_allocations;
+    RRDSET *st_avg_alloc;
+    RRDSET *st_ops;
+};
+
+static int do_memory_trace_item(void *item, void *data) {
+    struct memory_trace_data *tmp = data;
+    struct malloc_trace *p = item;
+
+    // ------------------------------------------------------------------------
+
+    if(!p->rd_bytes)
+        p->rd_bytes = rrddim_add(tmp->st_memory, p->function, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+    collected_number bytes = (collected_number)__atomic_load_n(&p->bytes, __ATOMIC_RELAXED);
+    rrddim_set_by_pointer(tmp->st_memory, p->rd_bytes, bytes);
+
+    // ------------------------------------------------------------------------
+
+    if(!p->rd_allocations)
+        p->rd_allocations = rrddim_add(tmp->st_allocations, p->function, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+    collected_number allocs = (collected_number)__atomic_load_n(&p->allocations, __ATOMIC_RELAXED);
+    rrddim_set_by_pointer(tmp->st_allocations, p->rd_allocations, allocs);
+
+    // ------------------------------------------------------------------------
+
+    if(!p->rd_avg_alloc)
+        p->rd_avg_alloc = rrddim_add(tmp->st_avg_alloc, p->function, NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+
+    collected_number avg_alloc = (allocs)?(bytes * 100 / allocs):0;
+    rrddim_set_by_pointer(tmp->st_avg_alloc, p->rd_avg_alloc, avg_alloc);
+
+    // ------------------------------------------------------------------------
+
+    if(!p->rd_ops)
+        p->rd_ops = rrddim_add(tmp->st_ops, p->function, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+    collected_number ops = 0;
+    ops += (collected_number)__atomic_load_n(&p->malloc_calls, __ATOMIC_RELAXED);
+    ops += (collected_number)__atomic_load_n(&p->calloc_calls, __ATOMIC_RELAXED);
+    ops += (collected_number)__atomic_load_n(&p->realloc_calls, __ATOMIC_RELAXED);
+    ops += (collected_number)__atomic_load_n(&p->strdup_calls, __ATOMIC_RELAXED);
+    ops += (collected_number)__atomic_load_n(&p->free_calls, __ATOMIC_RELAXED);
+    rrddim_set_by_pointer(tmp->st_ops, p->rd_ops, ops);
+
+    // ------------------------------------------------------------------------
+
+    return 1;
+}
+static void malloc_trace_statistics(void) {
+    static struct memory_trace_data tmp = {
+        .st_memory = NULL,
+        .st_allocations = NULL,
+        .st_avg_alloc = NULL,
+        .st_ops = NULL,
+    };
+
+    if(!tmp.st_memory) {
+        tmp.st_memory = rrdset_create_localhost(
+            "netdata"
+            , "memory_size"
+            , NULL
+            , "memory"
+            , "netdata.memory.size"
+            , "Netdata Memory Used by Function"
+            , "bytes"
+            , "netdata"
+            , "stats"
+            , 900000
+            , localhost->rrd_update_every
+            , RRDSET_TYPE_STACKED
+        );
+    }
+    else
+        rrdset_next(tmp.st_memory);
+
+    if(!tmp.st_ops) {
+        tmp.st_ops = rrdset_create_localhost(
+            "netdata"
+            , "memory_operations"
+            , NULL
+            , "memory"
+            , "netdata.memory.operations"
+            , "Netdata Memory Operations by Function"
+            , "ops/s"
+            , "netdata"
+            , "stats"
+            , 900001
+            , localhost->rrd_update_every
+            , RRDSET_TYPE_LINE
+        );
+    }
+    else
+        rrdset_next(tmp.st_ops);
+
+    if(!tmp.st_allocations) {
+        tmp.st_allocations = rrdset_create_localhost(
+            "netdata"
+            , "memory_allocations"
+            , NULL
+            , "memory"
+            , "netdata.memory.allocations"
+            , "Netdata Memory Allocations by Function"
+            , "allocations"
+            , "netdata"
+            , "stats"
+            , 900002
+            , localhost->rrd_update_every
+            , RRDSET_TYPE_STACKED
+        );
+    }
+    else
+        rrdset_next(tmp.st_allocations);
+
+    if(!tmp.st_avg_alloc) {
+        tmp.st_avg_alloc = rrdset_create_localhost(
+            "netdata"
+            , "memory_avg_alloc"
+            , NULL
+            , "memory"
+            , "netdata.memory.avg_alloc"
+            , "Netdata Average Allocation Size by Function"
+            , "bytes"
+            , "netdata"
+            , "stats"
+            , 900003
+            , localhost->rrd_update_every
+            , RRDSET_TYPE_LINE
+        );
+    }
+    else
+        rrdset_next(tmp.st_avg_alloc);
+
+    malloc_trace_walkthrough(do_memory_trace_item, &tmp);
+
+    rrdset_done(tmp.st_memory);
+    rrdset_done(tmp.st_ops);
+    rrdset_done(tmp.st_allocations);
+    rrdset_done(tmp.st_avg_alloc);
+}
+#endif
+
 static void dictionary_statistics(void) {
     for(int i = 0; dictionary_categories[i].stats ;i++) {
         update_dictionary_category_charts(&dictionary_categories[i]);
@@ -2375,6 +2523,7 @@ void *global_statistics_main(void *ptr)
     worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine");
     worker_register_job_name(WORKER_JOB_STRINGS, "strings");
     worker_register_job_name(WORKER_JOB_DICTIONARIES, "dictionaries");
+    worker_register_job_name(WORKER_JOB_MALLOC_TRACE, "malloc_trace");
 
     netdata_thread_cleanup_push(global_statistics_cleanup, ptr);
 
@@ -2404,8 +2553,10 @@ void *global_statistics_main(void *ptr)
         worker_is_busy(WORKER_JOB_REGISTRY);
         registry_statistics();
 
-        worker_is_busy(WORKER_JOB_DBENGINE);
-        dbengine_statistics_charts();
+        if(dbengine_enabled) {
+            worker_is_busy(WORKER_JOB_DBENGINE);
+            dbengine_statistics_charts();
+        }
 
         worker_is_busy(WORKER_JOB_HEARTBEAT);
         update_heartbeat_charts();
@@ -2415,6 +2566,11 @@ void *global_statistics_main(void *ptr)
 
         worker_is_busy(WORKER_JOB_DICTIONARIES);
         dictionary_statistics();
+
+#ifdef NETDATA_TRACE_ALLOCATIONS
+        worker_is_busy(WORKER_JOB_MALLOC_TRACE);
+        malloc_trace_statistics();
+#endif
     }
 
     netdata_thread_cleanup_pop(1);
diff --git a/daemon/main.c b/daemon/main.c
index 20b6068c87..8f3477ad97 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -55,13 +55,17 @@ void netdata_cleanup_and_exit(int ret) {
         // free the database
         info("EXIT: freeing database memory...");
 #ifdef ENABLE_DBENGINE
-        for(int tier = 0; tier < storage_tiers ; tier++)
-            rrdeng_prepare_exit(multidb_ctx[tier]);
+        if(dbengine_enabled) {
+            for (int tier = 0; tier < storage_tiers; tier++)
+                rrdeng_prepare_exit(multidb_ctx[tier]);
+        }
 #endif
         rrdhost_free_all();
 #ifdef ENABLE_DBENGINE
-        for(int tier = 0; tier < storage_tiers ; tier++)
-            rrdeng_exit(multidb_ctx[tier]);
+        if(dbengine_enabled) {
+            for (int tier = 0; tier < storage_tiers; tier++)
+                rrdeng_exit(multidb_ctx[tier]);
+        }
 #endif
     }
     sql_close_context_database();
@@ -255,7 +259,8 @@ void cancel_main_threads() {
 
     for (i = 0; static_threads[i].name != NULL ; i++)
         freez(static_threads[i].thread);
-    free(static_threads);
+
+    freez(static_threads);
 }
 
 struct option_def option_definitions[] = {
@@ -1001,6 +1006,8 @@ int main(int argc, char **argv) {
                             if(string_unittest(10000)) return 1;
                             if (dictionary_unittest(10000))
                                 return 1;
+                            if(aral_unittest(10000))
+                                return 1;
                             if (rrdlabels_unittest())
                                 return 1;
                             if (ctx_unittest())
@@ -1023,6 +1030,9 @@ int main(int argc, char **argv) {
                         else if(strcmp(optarg, "dicttest") == 0) {
                             return dictionary_unittest(10000);
                         }
+                        else if(strcmp(optarg, "araltest") == 0) {
+                            return aral_unittest(10000);
+                        }
                         else if(strcmp(optarg, "stringtest") == 0) {
                             return string_unittest(10000);
                         }
diff --git a/daemon/service.c b/daemon/service.c
index 71b377dbba..3a267402d1 100644
--- a/daemon/service.c
+++ b/daemon/service.c
@@ -216,10 +216,8 @@ restart_after_removal:
             info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid);
 
             if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)
-#ifdef ENABLE_DBENGINE
                 /* don't delete multi-host DB host files */
                 && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
-#endif
             ) {
                 worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);
                 rrdhost_delete_charts(host);
diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c
index 0e8f935ad7..95c3a46f80 100644
--- a/database/engine/journalfile.c
+++ b/database/engine/journalfile.c
@@ -527,7 +527,7 @@ int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfi
 
     info("Journal file \"%s\" loaded (size:%"PRIu64").", path, file_size);
     if (likely(journalfile->data))
-        munmap(journalfile->data, file_size);
+        netdata_munmap(journalfile->data, file_size);
     return 0;
 
     error:
diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c
index 97f3fe6e2f..ce808aadbc 100644
--- a/database/engine/rrdengine.c
+++ b/database/engine/rrdengine.c
@@ -30,7 +30,7 @@ void dbengine_page_free(void *page) {
     if (unlikely(db_engine_use_malloc))
         freez(page);
     else
-        munmap(page, RRDENG_BLOCK_SIZE);
+        netdata_munmap(page, RRDENG_BLOCK_SIZE);
 }
 
 static void sanity_check(void)
diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c
index 74674dbdbd..329f8590b8 100644
--- a/database/ram/rrddim_mem.c
+++ b/database/ram/rrddim_mem.c
@@ -16,7 +16,7 @@ void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused)
 STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
     RRDDIM *rd = (RRDDIM *)db_metric_handle;
     rd->db[rd->rrdset->current_entry] = pack_storage_number(NAN, SN_FLAG_NONE);
-    struct mem_collect_handle *ch = calloc(1, sizeof(struct mem_collect_handle));
+    struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
     ch->rd = rd;
     return (STORAGE_COLLECT_HANDLE *)ch;
 }
@@ -46,7 +46,7 @@ void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
 }
 
 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
-    free(collection_handle);
+    freez(collection_handle);
     return 0;
 }
 
@@ -142,7 +142,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_qu
     handle->rd = rd;
     handle->start_time = start_time;
     handle->end_time = end_time;
-    struct mem_query_handle* h = calloc(1, sizeof(struct mem_query_handle));
+    struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
     h->slot           = rrddim_time2slot(rd, start_time);
     h->last_slot      = rrddim_time2slot(rd, end_time);
     h->dt = rd->rrdset->update_every;
diff --git a/database/rrd.h b/database/rrd.h
index f72a1f4c69..c09709148b 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -52,6 +52,7 @@ struct pg_cache_page_index;
 #include "sqlite/sqlite_health.h"
 #include "rrdcontext.h"
 
+extern bool dbengine_enabled;
 extern int storage_tiers;
 extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
 
diff --git a/database/rrdcontext.c b/database/rrdcontext.c
index 9a9d4f1582..20f20597e6 100644
--- a/database/rrdcontext.c
+++ b/database/rrdcontext.c
@@ -2200,7 +2200,7 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) {
         max_last_time_t = rrddim_last_entry_t(rm->rrddim);
     }
 #ifdef ENABLE_DBENGINE
-    else {
+    else if (dbengine_enabled) {
         RRDHOST *rrdhost = rm->ri->rc->rrdhost;
         for (int tier = 0; tier < storage_tiers; tier++) {
             if(!rrdhost->storage_instance[tier]) continue;
@@ -2215,6 +2215,10 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) {
             }
         }
     }
+    else {
+        // cannot get retention
+        return;
+    }
 #endif
 
     if(min_first_time_t == LONG_MAX)
diff --git a/database/rrddim.c b/database/rrddim.c
index d5832a1dab..59ef31eed0 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -2,9 +2,6 @@
 
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
-#ifdef ENABLE_DBENGINE
-#include "database/engine/rrdengineapi.h"
-#endif
 #include "storage_engine.h"
 
 // ----------------------------------------------------------------------------
@@ -27,6 +24,12 @@ struct rrddim_constructor {
 
 };
 
+// isolated call to appear
+// separate in statistics
+static void *rrddim_alloc_db(size_t entries) {
+    return callocz(entries, sizeof(storage_number));
+}
+
 static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *constructor_data) {
     struct rrddim_constructor *ctr = constructor_data;
     RRDDIM *rd = rrddim;
@@ -73,7 +76,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
         size_t entries = st->entries;
         if(entries < 5) entries = 5;
 
-        rd->db = callocz(entries, sizeof(storage_number));
+        rd->db = rrddim_alloc_db(entries);
         rd->memsize = entries * sizeof(storage_number);
     }
 
@@ -222,7 +225,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
     if(rd->db) {
         if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
-            munmap(rd->db, rd->memsize);
+            netdata_munmap(rd->db, rd->memsize);
         else
             freez(rd->db);
     }
@@ -641,7 +644,7 @@ void rrddim_memory_file_free(RRDDIM *rd) {
 
     struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
     freez(rd_on_file->cache_filename);
-    munmap(rd_on_file, rd_on_file->memsize);
+    netdata_munmap(rd_on_file, rd_on_file->memsize);
 
     // remove the pointers from the RRDDIM
     rd->rd_on_file = NULL;
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 4b9aec96d3..de01aa948f 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -3,6 +3,7 @@
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
 
+bool dbengine_enabled = false; // will become true if and when dbengine is initialized
 int storage_tiers = 1;
 int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
 RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
@@ -328,12 +329,18 @@ RRDHOST *rrdhost_create(const char *hostname,
 ) {
     debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
 
+    rrd_check_wrlock();
+
+    if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) {
+        error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname);
+        memory_mode = RRD_MEMORY_MODE_ALLOC;
+    }
+
 #ifdef ENABLE_DBENGINE
     int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);
 #else
-    int is_legacy = 1;
+int is_legacy = 1;
 #endif
-    rrd_check_wrlock();
 
     int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
     RRDHOST *host = callocz(1, sizeof(RRDHOST));
@@ -384,8 +391,8 @@ RRDHOST *rrdhost_create(const char *hostname,
             host->cache_dir = strdupz(filename);
         }
 
-        if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || (
-           host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
+        if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||
+             (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
             int r = mkdir(host->cache_dir, 0775);
             if(r != 0 && errno != EEXIST)
                 error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
@@ -754,22 +761,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim
 // ----------------------------------------------------------------------------
 // RRDHOST global / startup initialization
 
-int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
-    rrdhost_init();
-
-    if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            fatal("Failed to initialize SQLite");
-        info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
-    }
-
-    if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
-        error_report("Failed to initialize context metadata database");
-    }
-
-    if (unlikely(!system_info))
-        goto unittest;
-
+void dbengine_init(char *hostname) {
 #ifdef ENABLE_DBENGINE
     storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
     if(storage_tiers < 1) {
@@ -857,7 +849,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
             error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", hostname, tier, tier);
             break;
         }
-        
+
         internal_error(true, "DBENGINE tier %d grouping iterations is set to %d", tier, storage_tiers_grouping_iterations[tier]);
         ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier);
         if(ret != 0) {
@@ -877,6 +869,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
     else if(!created_tiers)
         fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
 
+    dbengine_enabled = true;
 #else
     storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
     if(storage_tiers != 1) {
@@ -885,10 +878,49 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
         config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
     }
 #endif
+}
+
+int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
+    rrdhost_init();
+
+    if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            fatal("Failed to initialize SQLite");
+        info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
+    }
+
+    if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
+        error_report("Failed to initialize context metadata database");
+    }
+
+    if (unlikely(strcmp(hostname, "unittest") == 0)) {
+        dbengine_enabled = true;
+        goto unittest;
+    }
 
-    health_init();
     rrdpush_init();
 
+    if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || storage_tiers > 1 || rrdpush_receiver_needs_dbengine()) {
+        info("Initializing dbengine...");
+        dbengine_init(hostname);
+    }
+    else
+        info("Not initializing dbengine...");
+
+    if(!dbengine_enabled) {
+        if (storage_tiers > 1) {
+            error("dbengine is not enabled, but %d tiers have been requested. Resetting tiers to 1", storage_tiers);
+            storage_tiers = 1;
+        }
+
+        if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+            error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
+            default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
+        }
+    }
+
+    health_init();
+
 unittest:
     debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
     rrd_wrlock();
@@ -1418,10 +1450,8 @@ void rrdhost_cleanup_all(void) {
     RRDHOST *host;
     rrdhost_foreach_read(host) {
         if (host != localhost && rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST) && !host->receiver
-#ifdef ENABLE_DBENGINE
             /* don't delete multi-host DB host files */
             && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0]))
-#endif
         )
             rrdhost_delete_charts(host);
         else
diff --git a/database/rrdset.c b/database/rrdset.c
index 426582ca0e..9b04712afb 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -1972,7 +1972,7 @@ void rrdset_memory_file_free(RRDSET *st) {
     rrdset_memory_file_update(st);
 
     struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
-    munmap(st_on_file, st_on_file->memsize);
+    netdata_munmap(st_on_file, st_on_file->memsize);
 
     // remove the pointers from the RRDDIM
     st->st_on_file = NULL;
diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c
index 3e34f664cb..03edd6bc8a 100644
--- a/database/sqlite/sqlite_aclk.c
+++ b/database/sqlite/sqlite_aclk.c
@@ -264,13 +264,6 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
 
     struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
     system_info->hops = str2i((const char *) argv[IDX_HOPS]);
-    RRD_MEMORY_MODE memory_mode;
-
-#ifdef ENABLE_DBENGINE
-    memory_mode = RRD_MEMORY_MODE_DBENGINE;
-#else
-    memory_mode = RRD_MEMORY_MODE_RAM;
-#endif
 
     sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);
 
@@ -287,7 +280,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
         , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")
         , argv[3] ? str2i(argv[IDX_UPDATE_EVERY]) : 1
         , argv[13] ? str2i(argv[IDX_ENTRIES]) : 0
-        , memory_mode
+        , default_rrd_memory_mode
         , 0 // health
         , 0 // rrdpush enabled
         , NULL  //destination
diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c
index b7ed5db103..4137a3baa4 100644
--- a/database/sqlite/sqlite_functions.c
+++ b/database/sqlite/sqlite_functions.c
@@ -1408,8 +1408,10 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
     rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
 
 #ifdef ENABLE_DBENGINE
-    for(int tier = 0; tier < storage_tiers ; tier++)
-        host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
+    if(dbengine_enabled) {
+        for (int tier = 0; tier < storage_tiers; tier++)
+            host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
+    }
 #endif
 
 failed:
diff --git a/libnetdata/arrayalloc/arrayalloc.c b/libnetdata/arrayalloc/arrayalloc.c
index c3e8114a83..920669d931 100644
--- a/libnetdata/arrayalloc/arrayalloc.c
+++ b/libnetdata/arrayalloc/arrayalloc.c
@@ -162,7 +162,11 @@ static inline ARAL_PAGE *find_page_with_allocation(ARAL *ar, void *ptr) {
     return page;
 }
 
-static void arrayalloc_increase(ARAL *ar) {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+static void arrayalloc_add_page(ARAL *ar, const char *file, const char *function, size_t line) {
+#else
+static void arrayalloc_add_page(ARAL *ar) {
+#endif
     if(unlikely(!ar->internal.initialized))
         arrayalloc_init(ar);
 
@@ -182,8 +186,13 @@ static void arrayalloc_increase(ARAL *ar) {
         if (unlikely(!page->data))
             fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename);
     }
-    else
+    else {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+        page->data = mallocz_int(page->size, file, function, line);
+#else
         page->data = mallocz(page->size);
+#endif
+    }
 
     // link the free space to its page
     ARAL_FREE *fr = (ARAL_FREE *)page->data;
@@ -217,14 +226,23 @@ ARAL *arrayalloc_create(size_t element_size, size_t elements, const