full memory tracking and profiling of Netdata Agent (#13789)

* full memory tracking and profiling of Netdata Agent

* initialize dbengine only when it is needed

* handling of dbengine compiled but not available

* restore unittest

* restore unittest again

* more improvements about ifdef dbengine

* fix compilation when dbengine is not enabled

* check if dbengine is enabled on exit

* call freez() not free()

* aral unittest

* internal checks activate trace allocations; dev mode activates internal checks

8 files changed, 480 insertions, 193 deletions

diff --git a/libnetdata/arrayalloc/arrayalloc.c b/libnetdata/arrayalloc/arrayalloc.c
index c3e8114a83..920669d931 100644
--- a/libnetdata/arrayalloc/arrayalloc.c
+++ b/libnetdata/arrayalloc/arrayalloc.c
@@ -162,7 +162,11 @@ static inline ARAL_PAGE *find_page_with_allocation(ARAL *ar, void *ptr) {
     return page;
 }
 
-static void arrayalloc_increase(ARAL *ar) {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+static void arrayalloc_add_page(ARAL *ar, const char *file, const char *function, size_t line) {
+#else
+static void arrayalloc_add_page(ARAL *ar) {
+#endif
     if(unlikely(!ar->internal.initialized))
         arrayalloc_init(ar);
 
@@ -182,8 +186,13 @@ static void arrayalloc_increase(ARAL *ar) {
         if (unlikely(!page->data))
             fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename);
     }
-    else
+    else {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+        page->data = mallocz_int(page->size, file, function, line);
+#else
         page->data = mallocz(page->size);
+#endif
+    }
 
     // link the free space to its page
     ARAL_FREE *fr = (ARAL_FREE *)page->data;
@@ -217,14 +226,23 @@ ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filena
     return ar;
 }
 
+#ifdef NETDATA_TRACE_ALLOCATIONS
+void *arrayalloc_mallocz_int(ARAL *ar, const char *file, const char *function, size_t line) {
+#else
 void *arrayalloc_mallocz(ARAL *ar) {
+#endif
     if(unlikely(!ar->internal.initialized))
         arrayalloc_init(ar);
 
     arrayalloc_lock(ar);
 
-    if(unlikely(!ar->internal.first_page || !ar->internal.first_page->free_list))
-        arrayalloc_increase(ar);
+    if(unlikely(!ar->internal.first_page || !ar->internal.first_page->free_list)) {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+        arrayalloc_add_page(ar, file, function, line);
+#else
+        arrayalloc_add_page(ar);
+#endif
+    }
 
     ARAL_PAGE *page = ar->internal.first_page;
     ARAL_FREE *fr = page->free_list;
@@ -266,7 +284,11 @@ void *arrayalloc_mallocz(ARAL *ar) {
     return (void *)fr;
 }
 
+#ifdef NETDATA_TRACE_ALLOCATIONS
+void arrayalloc_freez_int(ARAL *ar, void *ptr, const char *file, const char *function, size_t line) {
+#else
 void arrayalloc_freez(ARAL *ar, void *ptr) {
+#endif
     if(!ptr) return;
     arrayalloc_lock(ar);
 
@@ -319,13 +341,18 @@ void arrayalloc_freez(ARAL *ar, void *ptr) {
 
         // free it
         if(ar->internal.mmap) {
-            munmap(page->data, page->size);
+            netdata_munmap(page->data, page->size);
             if (unlikely(unlink(page->filename) == 1))
                 error("Cannot delete file '%s'", page->filename);
             freez((void *)page->filename);
         }
-        else
+        else {
+#ifdef NETDATA_TRACE_ALLOCATIONS
+            freez_int(page->data, file, function, line);
+#else
             freez(page->data);
+#endif
+        }
 
         freez(page);
     }
@@ -336,3 +363,89 @@ void arrayalloc_freez(ARAL *ar, void *ptr) {
 
     arrayalloc_unlock(ar);
 }
+
+int aral_unittest(size_t elements) {
+    char *cache_dir = "/tmp/";
+    ARAL *ar = arrayalloc_create(20, 10, "test-aral", &cache_dir);
+    ar->use_mmap = false;
+
+    void *pointers[elements];
+
+    for(size_t i = 0; i < elements ;i++) {
+        pointers[i] = arrayalloc_mallocz(ar);
+    }
+
+    for(size_t div = 5; div >= 2 ;div--) {
+        for (size_t i = 0; i < elements / div; i++) {
+            arrayalloc_freez(ar, pointers[i]);
+        }
+
+        for (size_t i = 0; i < elements / div; i++) {
+            pointers[i] = arrayalloc_mallocz(ar);
+        }
+    }
+
+    for(size_t step = 50; step >= 10 ;step -= 10) {
+        for (size_t i = 0; i < elements; i += step) {
+            arrayalloc_freez(ar, pointers[i]);
+        }
+
+        for (size_t i = 0; i < elements; i += step) {
+            pointers[i] = arrayalloc_mallocz(ar);
+        }
+    }
+
+    for(size_t i = 0; i < elements ;i++) {
+        arrayalloc_freez(ar, pointers[i]);
+    }
+
+    if(ar->internal.first_page) {
+        fprintf(stderr, "ARAL leftovers detected (1)");
+        return 1;
+    }
+
+    size_t ops = 0;
+    size_t increment = elements / 10;
+    size_t allocated = 0;
+    for(size_t all = increment; all <= elements ; all += increment) {
+
+        for(; allocated < all ; allocated++) {
+            pointers[allocated] = arrayalloc_mallocz(ar);
+            ops++;
+        }
+
+        size_t to_free = now_realtime_usec() % all;
+        size_t free_list[to_free];
+        for(size_t i = 0; i < to_free ;i++) {
+            size_t pos;
+            do {
+                pos = now_realtime_usec() % all;
+            } while(!pointers[pos]);
+
+            arrayalloc_freez(ar, pointers[pos]);
+            pointers[pos] = NULL;
+            free_list[i] = pos;
+            ops++;
+        }
+
+        for(size_t i = 0; i < to_free ;i++) {
+            size_t pos = free_list[i];
+            pointers[pos] = arrayalloc_mallocz(ar);
+            ops++;
+        }
+    }
+
+    for(size_t i = 0; i < allocated - 1 ;i++) {
+        arrayalloc_freez(ar, pointers[i]);
+        ops++;
+    }
+
+    arrayalloc_freez(ar, pointers[allocated - 1]);
+
+    if(ar->internal.first_page) {
+        fprintf(stderr, "ARAL leftovers detected (2)");
+        return 1;
+    }
+
+    return 0;
+}
diff --git a/libnetdata/arrayalloc/arrayalloc.h b/libnetdata/arrayalloc/arrayalloc.h
index d04c7c74a9..66a2832e80 100644
--- a/libnetdata/arrayalloc/arrayalloc.h
+++ b/libnetdata/arrayalloc/arrayalloc.h
@@ -29,7 +29,21 @@ typedef struct arrayalloc {
 } ARAL;
 
 ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filename, char **cache_dir);
+int aral_unittest(size_t elements);
+
+#ifdef NETDATA_TRACE_ALLOCATIONS
+
+#define arrayalloc_mallocz(ar) arrayalloc_mallocz_int(ar, __FILE__, __FUNCTION__, __LINE__)
+#define arrayalloc_freez(ar, ptr) arrayalloc_freez_int(ar, ptr, __FILE__, __FUNCTION__, __LINE__)
+
+void *arrayalloc_mallocz_int(ARAL *ar, const char *file, const char *function, size_t line);
+void arrayalloc_freez_int(ARAL *ar, void *ptr, const char *file, const char *function, size_t line);
+
+#else // NETDATA_TRACE_ALLOCATIONS
+
 void *arrayalloc_mallocz(ARAL *ar);
 void arrayalloc_freez(ARAL *ar, void *ptr);
 
+#endif // NETDATA_TRACE_ALLOCATIONS
+
 #endif // ARRAYALLOC_H
diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c
index ba1f059d57..938c7dde27 100644
--- a/libnetdata/config/appconfig.c
+++ b/libnetdata/config/appconfig.c
@@ -473,7 +473,7 @@ NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, con
     return str2ndd(s, NULL);
 }
 
-static inline int appconfig_test_boolean_value(char *s) {
+inline int appconfig_test_boolean_value(char *s) {
     if(!strcasecmp(s, "yes") || !strcasecmp(s, "true") || !strcasecmp(s, "on")
        || !strcasecmp(s, "auto") || !strcasecmp(s, "on demand"))
         return 1;
diff --git a/libnetdata/config/appconfig.h b/libnetdata/config/appconfig.h
index 7707860ded..2828e107a5 100644
--- a/libnetdata/config/appconfig.h
+++ b/libnetdata/config/appconfig.h
@@ -199,6 +199,8 @@ struct section *appconfig_get_section(struct config *root, const char *name);
 void appconfig_wrlock(struct config *root);
 void appconfig_unlock(struct config *root);
 
+int appconfig_test_boolean_value(char *s);
+
 struct connector_instance {
     char instance_name[CONFIG_MAX_NAME + 1];
     char connector_name[CONFIG_MAX_NAME + 1];
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
index 9d19d5965d..98a9c5a2a8 100644
--- a/libnetdata/dictionary/dictionary.c
+++ b/libnetdata/dictionary/dictionary.c
@@ -173,7 +173,7 @@ struct dictionary {
 // forward definitions of functions used in reverse order in the code
 static void garbage_collect_pending_deletes(DICTIONARY *dict);
 static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item);
-static size_t item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item);
+static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item);
 static inline const char *item_get_name(const DICTIONARY_ITEM *item);
 static bool item_is_not_referenced_and_can_be_removed(DICTIONARY *dict, DICTIONARY_ITEM *item);
 static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *item);
@@ -688,7 +688,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
 
             if(item_is_not_referenced_and_can_be_removed(dict, item)) {
                 DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next);
-                item_free_with_hooks(dict, item);
+                dict_item_free_with_hooks(dict, item);
                 deleted++;
 
                 pending = DICTIONARY_PENDING_DELETES_MINUS1(dict);
@@ -1075,7 +1075,7 @@ static inline const char *item_get_name(const DICTIONARY_ITEM *item) {
         return item->caller_name;
 }
 
-static DICTIONARY_ITEM *item_allocate(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) {
+static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) {
     DICTIONARY_ITEM *item;
 
     size_t size = sizeof(DICTIONARY_ITEM);
@@ -1102,7 +1102,29 @@ static DICTIONARY_ITEM *item_allocate(DICTIONARY *dict __maybe_unused, size_t *a
     return item;
 }
 
-static DICTIONARY_ITEM *item_create_with_hooks(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
+static void *dict_item_value_create(void *value, size_t value_len) {
+    void *ptr = NULL;
+
+    if(likely(value_len)) {
+        if (likely(value)) {
+            // a value has been supplied
+            // copy it
+            ptr =  mallocz(value_len);
+            memcpy(ptr, value, value_len);
+        }
+        else {
+            // no value has been supplied
+            // allocate a clear memory block
+            ptr = callocz(1, value_len);
+        }
+    }
+    // else
+    // the caller wants an item without any value
+
+    return ptr;
+}
+
+static DICTIONARY_ITEM *dict_item_create_with_hooks(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
 #ifdef NETDATA_INTERNAL_CHECKS
     if(unlikely(name_len > KEY_LEN_MAX))
         fatal("DICTIONARY: tried to index a key of size %zu, but the maximum acceptable is %zu", name_len, (size_t)KEY_LEN_MAX);
@@ -1113,7 +1135,7 @@ static DICTIONARY_ITEM *item_create_with_hooks(DICTIONARY *dict, const char *nam
 
     size_t item_size = 0, key_size = 0, value_size = 0;
 
-    DICTIONARY_ITEM *item = item_allocate(dict, &item_size, master_item);
+    DICTIONARY_ITEM *item = dict_item_create(dict, &item_size, master_item);
     key_size += item_set_name(dict, item, name, name_len);
 
     if(unlikely(is_view_dictionary(dict))) {
@@ -1129,28 +1151,11 @@ static DICTIONARY_ITEM *item_create_with_hooks(DICTIONARY *dict, const char *nam
     else {
         // we are on the master dictionary
 
-        if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))
+        if(unlikely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))
             item->shared->value = value;
-        else {
-            if(likely(value_len)) {
-                if(value) {
-                    // a value has been supplied
-                    // copy it
-                    item->shared->value = mallocz(value_len);
-                    memcpy(item->shared->value, value, value_len);
-                }
-                else {
-                    // no value has been supplied
-                    // allocate a clear memory block
-                    item->shared->value = callocz(1, value_len);
-                }
+        else
+            item->shared->value = dict_item_value_create(value, value_len);
 
-            }
-            else {
-                // the caller wants an item without any value
-                item->shared->value = NULL;
-            }
-        }
         item->shared->value_len = value_len;
         value_size += value_len;
 
@@ -1163,7 +1168,7 @@ static DICTIONARY_ITEM *item_create_with_hooks(DICTIONARY *dict, const char *nam
     return item;
 }
 
-static void item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item, void *value, size_t value_len, void *constructor_data) {
+static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item, void *value, size_t value_len, void *constructor_data) {
     if(unlikely(is_view_dictionary(dict)))
         fatal("DICTIONARY: %s() should never be called on views.", __FUNCTION__ );
 
@@ -1203,7 +1208,7 @@ static void item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item,
     dictionary_execute_insert_callback(dict, item, constructor_data);
 }
 
-static size_t item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) {
+static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) {
     debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item));
 
     size_t item_size = 0, key_size = 0, value_size = 0;
@@ -1239,7 +1244,7 @@ static size_t item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) {
 // ----------------------------------------------------------------------------
 // item operations
 
-static void item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
+static void dict_item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
     if(is_master_dictionary(dict)) {
         item_shared_flag_set(item, ITEM_FLAG_DELETED);
 
@@ -1248,14 +1253,14 @@ static void item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
     }
 }
 
-static inline void item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
+static inline void dict_item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
     if(item_is_not_referenced_and_can_be_removed(dict, item)) {
-        item_shared_set_deleted(dict, item);
+        dict_item_shared_set_deleted(dict, item);
         item_linked_list_remove(dict, item);
-        item_free_with_hooks(dict, item);
+        dict_item_free_with_hooks(dict, item);
     }
     else {
-        item_shared_set_deleted(dict, item);
+        dict_item_shared_set_deleted(dict, item);
         item_flag_set(item, ITEM_FLAG_DELETED);
         // after this point do not touch the item
     }
@@ -1269,7 +1274,7 @@ static inline void item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *
 // the need for the garbage collector to kick-in later.
 // Most deletions happen during traversal, so this is a nice hack
 // to speed up everything!
-static inline void item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(DICTIONARY *dict, DICTIONARY_ITEM *item, char rw) {
+static inline void dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(DICTIONARY *dict, DICTIONARY_ITEM *item, char rw) {
     if(rw == DICTIONARY_LOCK_WRITE) {
         bool should_be_deleted = item_flag_check(item, ITEM_FLAG_DELETED);
 
@@ -1281,7 +1286,7 @@ static inline void item_release_and_check_if_it_is_deleted_and_can_be_removed_un
             DICTIONARY_PENDING_DELETES_MINUS1(dict);
 
             item_linked_list_remove(dict, item);
-            item_free_with_hooks(dict, item);
+            dict_item_free_with_hooks(dict, item);
         }
     }
     else {
@@ -1290,7 +1295,7 @@ static inline void item_release_and_check_if_it_is_deleted_and_can_be_removed_un
     }
 }
 
-static bool item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
+static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
     if(unlikely(!name || !*name)) {
         internal_error(
             true,
@@ -1330,14 +1335,14 @@ static bool item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
 
         dictionary_index_lock_unlock(dict);
 
-        item_free_or_mark_deleted(dict, item);
+        dict_item_free_or_mark_deleted(dict, item);
         ret = true;
     }
 
     return ret;
 }
 
-static DICTIONARY_ITEM *item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
+static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
     if(unlikely(!name || !*name)) {
         internal_error(
             true,
@@ -1381,7 +1386,8 @@ static DICTIONARY_ITEM *item_add_or_reset_value_and_acquire(DICTIONARY *dict, co
             // a new item added to the index
 
             // create the dictionary item
-            item = *item_pptr = item_create_with_hooks(dict, name, name_len, value, value_len, constructor_data, master_item);
+            item = *item_pptr =
+                dict_item_create_with_hooks(dict, name, name_len, value, value_len, constructor_data, master_item);
 
             // call the hashtable react
             hashtable_inserted_item_unsafe(dict, item);
@@ -1420,7 +1426,7 @@ static DICTIONARY_ITEM *item_add_or_reset_value_and_acquire(DICTIONARY *dict, co
                 // the user wants to reset its value
 
                 if (!(dict->options & DICT_OPTION_DONT_OVERWRITE_VALUE)) {
-                    item_reset_value_with_hooks(dict, item, value, value_len, constructor_data);
+                    dict_item_reset_value_with_hooks(dict, item, value, value_len, constructor_data);
                     added_or_updated = true;
                 }
 
@@ -1449,7 +1455,7 @@ static DICTIONARY_ITEM *item_add_or_reset_value_and_acquire(DICTIONARY *dict, co
     return item;
 }
 
-static DICTIONARY_ITEM *item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) {
+static DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) {
     if(unlikely(!name || !*name)) {
         internal_error(
             true,
@@ -1517,7 +1523,7 @@ static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool fo
         // cache item->next
         // because we are going to free item
         DICTIONARY_ITEM *item_next = item->next;
-        item_size += item_free_with_hooks(dict, item);
+        item_size += dict_item_free_with_hooks(dict, item);
         item = item_next;
 
         DICTIONARY_ENTRIES_MINUS1(dict);
@@ -1800,7 +1806,7 @@ void dictionary_flush(DICTIONARY *dict) {
         item_next = item->next;
 
         if(!item_flag_check(item, ITEM_FLAG_DELETED))
-            item_free_or_mark_deleted(dict, item);
+            dict_item_free_or_mark_deleted(dict, item);
     }
     ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
 
@@ -1850,7 +1856,8 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_set_and_acquire_item_advanced(DICTIO
     if(unlikely(is_view_dictionary(dict)))
         fatal("DICTIONARY: this dictionary is a view, you cannot add items other than the ones from the master dictionary.");
 
-    DICTIONARY_ITEM *item = item_add_or_reset_value_and_acquire(dict, name, name_len, value, value_len, constructor_data, NULL);
+    DICTIONARY_ITEM *item =
+        dict_item_add_or_reset_value_and_acquire(dict, name, name_len, value, value_len, constructor_data, NULL);
     api_internal_check(dict, item, false, false);
     return item;
 }
@@ -1877,7 +1884,7 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(D
         fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries.");
 
     dictionary_acquired_item_dup(dict->master, master_item);
-    DICTIONARY_ITEM *item = item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
+    DICTIONARY_ITEM *item = dict_item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
     dictionary_acquired_item_release(dict->master, master_item);
 
     api_internal_check(dict, item, false, false);
@@ -1904,7 +1911,7 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_get_and_acquire_item_advanced(DICTIO
         return NULL;
 
     api_internal_check(dict, NULL, false, true);
-    DICTIONARY_ITEM *item = item_find_and_acquire(dict, name, name_len);
+    DICTIONARY_ITEM *item = dict_item_find_and_acquire(dict, name, name_len);
     api_internal_check(dict, item, false, true);
     return item;
 }
@@ -1977,7 +1984,7 @@ bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_le
         return false;
 
     api_internal_check(dict, NULL, false, true);
-    return item_del(dict, name, name_len);
+    return dict_item_del(dict, name, name_len);
 }
 
 // ----------------------------------------------------------------------------
@@ -2052,7 +2059,7 @@ void *dictionary_foreach_next(DICTFE *dfe) {
         item_next = item_next->next;
 
     if(likely(item)) {
-        item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
+        dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
         // item_release(dfe->dict, item);
     }
 
@@ -2088,7 +2095,7 @@ void dictionary_foreach_done(DICTFE *dfe) {
 
     // release it, so that it can possibly be deleted
     if(likely(item)) {
-        item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
+        dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
         // item_release(dfe->dict, item);
     }
 
@@ -2143,7 +2150,7 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const D
         // until we release the reference counter, so the pointers are there
         item_next = item->next;
 
-        item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
+        dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
         // item_release(dict, item);
 
         if(unlikely(r < 0)) {
@@ -2203,7 +2210,7 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
         if(callit)
             r = callback(item, item->shared->value, data);
 
-        item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
+        dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
         // item_release(dict, item);
 
         if(r < 0) {
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index ee5e4851da..16781f39bb 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -30,128 +30,237 @@ const char *program_version = VERSION;
 // its lifetime), these can be used to override the default system allocation
 // routines.
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-#warning NETDATA_LOG_ALLOCATIONS ENABLED - set log_thread_memory_allocations=1 on any thread to log all its allocations - or use log_allocations() to log them on demand
-
-static __thread struct memory_statistics {
-    volatile ssize_t malloc_calls_made;
-    volatile ssize_t calloc_calls_made;
-    volatile ssize_t realloc_calls_made;
-    volatile ssize_t strdup_calls_made;
-    volatile ssize_t free_calls_made;
-    volatile ssize_t memory_calls_made;
-    volatile ssize_t allocated_memory;
-    volatile ssize_t mmapped_memory;
-} memory_statistics = { 0, 0, 0, 0, 0, 0, 0, 0 };
-
-__thread size_t log_thread_memory_allocations = 0;
-
-inline void log_allocations_int(const char *file, const char *function, const unsigned long line) {
-    static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
-
-    fprintf(stderr, "%s MEMORY ALLOCATIONS: (%04lu@%s:%s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
-            netdata_thread_tag(),
-            line, file, function,
-            (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
-            (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
-            memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
-            memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
-            memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
-            memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
-            memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
-    );
-
-    memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
-}
-
-static inline void mmap_accounting(size_t size) {
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.mmapped_memory += size;
-    }
+#ifdef NETDATA_TRACE_ALLOCATIONS
+#warning NETDATA_TRACE_ALLOCATIONS ENABLED
+#include "Judy.h"
+
+Word_t JudyMalloc(Word_t Words) {
+    Word_t Addr;
+
+    Addr = (Word_t) mallocz(Words * sizeof(Word_t));
+    return(Addr);
+}
+void JudyFree(void * PWord, Word_t Words) {
+    (void)Words;
+    freez(PWord);
+}
+Word_t JudyMallocVirtual(Word_t Words) {
+    Word_t Addr;
+
+    Addr = (Word_t) mallocz(Words * sizeof(Word_t));
+    return(Addr);
+}
+void JudyFreeVirtual(void * PWord, Word_t Words) {
+    (void)Words;
+    freez(PWord);
 }
 
-void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
-    memory_statistics.memory_calls_made++;
-    memory_statistics.malloc_calls_made++;
-    memory_statistics.allocated_memory += size;
+#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
+#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
+#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - (size % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
 
-    if(log_thread_memory_allocations)
-        log_allocations_int(file, function, line);
+struct malloc_header_signature {
+    uint32_t magic;
+    uint32_t size;
+    struct malloc_trace *trace;
+};
+
+struct malloc_header {
+    struct malloc_header_signature signature;
+    uint8_t padding[(sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) ? MALLOC_ALIGNMENT - (sizeof(struct malloc_header_signature) % MALLOC_ALIGNMENT) : 0];
+    uint8_t data[];
+};
+
+static size_t malloc_header_size = sizeof(struct malloc_header);
 
-    size_t *n = (size_t *)malloc(sizeof(size_t) + size);
-    if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
-    *n = size;
-    return (void *)&n[1];
+int malloc_trace_compare(void *A, void *B) {
+    struct malloc_trace *a = A;
+    struct malloc_trace *b = B;
+    return strcmp(a->function, b->function);
 }
 
-void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
-    size = nmemb * size;
+static avl_tree_lock malloc_trace_index = {
+    .avl_tree = {
+        .root = NULL,
+        .compar = malloc_trace_compare},
+    .rwlock = NETDATA_RWLOCK_INITIALIZER
+};
+
+int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) {
+    return avl_traverse_lock(&malloc_trace_index, callback, data);
+}
+
+NEVERNULL WARNUNUSED
+static struct malloc_trace *malloc_trace_find_or_create(const char *file, const char *function, size_t line) {
+    struct malloc_trace tmp = {
+        .line = line,
+        .function = function,
+        .file = file,
+    };
+
+    struct malloc_trace *t = (struct malloc_trace *)avl_search_lock(&malloc_trace_index, (avl_t *)&tmp);
+    if(!t) {
+        t = calloc(1, sizeof(struct malloc_trace));
+        if(!t) fatal("No memory");
+        t->line = line;
+        t->function = function;
+        t->file = file;
+
+        struct malloc_trace *t2 = (struct malloc_trace *)avl_insert_lock(&malloc_trace_index, (avl_t *)t);
+        if(t2 != t)
+            free(t);
+
+        t = t2;
+    }
+
+    if(!t)
+        fatal("Cannot insert to AVL");
+
+    return t;
+}
 
-    memory_statistics.memory_calls_made++;
-    memory_statistics.calloc_calls_made++;
-    memory_statistics.allocated_memory += size;
-    if(log_thread_memory_allocations)
-        log_allocations_int(file, function, line);
+void malloc_trace_mmap(size_t size) {
+    struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);
+    size_t_atomic_count(add, p->mmap_calls, 1);
+    size_t_atomic_count(add, p->allocations, 1);
+    size_t_atomic_bytes(add, p->bytes, size);
+}
 
-    size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
-    if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
-    *n = size;
-    return (void *)&n[1];
+void malloc_trace_munmap(size_t size) {
+    struct malloc_trace *p = malloc_trace_find_or_create("unknown", "netdata_mmap", 1);
+    size_t_atomic_count(add, p->munmap_calls, 1);
+    size_t_atomic_count(sub, p->allocations, 1);
+    size_t_atomic_bytes(sub, p->bytes, size);
 }
 
-void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
-    if(!ptr) return mallocz_int(file, function, line, size);
+void *mallocz_int(size_t size, const char *file, const char *function, size_t line) {
+    struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
 
-    size_t *n = (size_t *)ptr;
-    n--;
-    size_t old_size = *n;
+    size_t_atomic_count(add, p->malloc_calls, 1);
+    size_t_atomic_count(add, p->allocations, 1);
+    size_t_atomic_bytes(add, p->bytes, size);
 
-    n = realloc(n, sizeof(size_t) + size);
-    if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
+    struct malloc_header *t = (struct malloc_header *)malloc(malloc_header_size + size);
+    if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
+    t->signature.magic = 0x0BADCAFE;
+    t->signature.trace = p;
+    t->signature.size = size;
 
-    memory_statistics.memory_calls_made++;
-    memory_statistics.realloc_calls_made++;
-    memory_statistics.allocated_memory += (size - old_size);
-    if(log_thread_memory_allocations)
-        log_allocations_int(file, function, line);
+#ifdef NETDATA_INTERNAL_CHECKS
+    for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
+        t->padding[i] = 0xFF;
+#endif
 
-    *n = size;
-    return (void *)&n[1];
+    return (void *)&t->data;
 }
 
-char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
+void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line) {
+    struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
+    size = nmemb * size;
+
+    size_t_atomic_count(add, p->calloc_calls, 1);
+    size_t_atomic_count(add, p->allocations, 1);
+    size_t_atomic_bytes(add, p->bytes, size);
+
+    struct malloc_header *t = (struct malloc_header *)calloc(1, malloc_header_size + size);
+    if (unlikely(!t)) fatal("mallocz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
+    t->signature.magic = 0x0BADCAFE;
+    t->signature.trace = p;
+    t->signature.size = size;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+    for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
+        t->padding[i] = 0xFF;
+#endif
+
+    return &t->data;
+}
+
+char *strdupz_int(const char *s, const char *file, const char *function, size_t line) {
+    struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
     size_t size = strlen(s) + 1;
 
-    memory_statistics.memory_calls_made++;
-    memory_statistics.strdup_calls_made++;
-    memory_statistics.allocated_memory += size;
-    if(log_thread_memory_allocations)
-        log_allocations_int(file, function, line);
+    size_t_atomic_count(add, p->strdup_calls, 1);
+    size_t_atomic_count(add, p->allocations, 1);
+    size_t_atomic_bytes(add, p->bytes, size);
+
+    struct malloc_header *t = (struct malloc_header *)malloc(malloc_header_size + size);
+    if (unlikely(!t)) fatal("strdupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
+    t->signature.magic = 0x0BADCAFE;
+    t->signature.trace = p;
+    t->signature.size = size;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+    for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
+        t->padding[i] = 0xFF;
+#endif
+
+    strcpy((char *)&t->data, s);
+    return (char *)&t->data;
+}
+
+static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {
+    uint8_t *ret = (uint8_t *)ptr - malloc_header_size;
+    struct malloc_header *t = (struct malloc_header *)ret;
 
-    size_t *n = (size_t *)malloc(sizeof(size_t) + size);
-    if (unlikely(!n)) f