From 7f8f11eb373dfc7bf6ac5a03e57a1b03487a279e Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 30 Jan 2023 20:36:16 +0200 Subject: DBENGINE v2 - improvements part 11 (#14337) * acquiring / releasing interface for metrics * metrics registry statistics * cleanup metrics registry by deleting metrics when they dont have retention anymore; do not double copy the data of pages to be flushed * print the tier in retention summary * Open files with buffered instead of direct I/O (test) * added more metrics stats and fixed unittest * rename writer functions to avoid confusion with refcounting * do not release a metric that is not acquired * Revert to use direct I/O on write -- use direct I/O on read as well * keep track of ARAL overhead and add it to the memory chart * aral full check via api * Cleanup * give names to ARALs and PGCs * aral improvements * restore query expansion to the future * prefer higher resolution tier when switching plans * added extent read statistics * smoother joining of tiers at query engine * fine tune aral max allocation size * aral restructuring to hide its internals from the rest of netdata * aral restructuring; addtion of defrag option to aral to keep the linked list sorted - enabled by default to test it * fully async aral * some statistics and cleanup * fix infinite loop while calculating retention * aral docs and defragmenting disabled by default * fix bug and add optimization when defragmenter is not enabled * aral stress test * aral speed report and documentation * added internal checks that all pages are full * improve internal log about metrics deletion * metrics registry uses one aral per partition * metrics registry aral max size to 512 elements per page * remove data_structures/README.md dependency --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> --- libnetdata/dictionary/dictionary.c | 60 ++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 22 deletions(-) (limited to 'libnetdata/dictionary/dictionary.c') diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index acc509e7ce..cbe6d0368e 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -789,7 +789,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) { // we didn't get a reference if(item_is_not_referenced_and_can_be_removed(dict, item)) { - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next); dict_item_free_with_hooks(dict, item); deleted++; @@ -1167,9 +1167,9 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); if(dict->options & DICT_OPTION_ADD_IN_FRONT) - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(dict->items.list, item, prev, next); else - DOUBLE_LINKED_LIST_APPEND_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS item->ll_adder_pid = gettid(); @@ -1186,7 +1186,7 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) { ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS item->ll_remover_pid = gettid(); @@ -1234,28 +1234,43 @@ static inline size_t item_get_name_len(const DICTIONARY_ITEM *item) { return strlen(item->caller_name); } -static ARAL dict_items_aral = { - .filename = NULL, - .cache_dir = NULL, - .use_mmap = false, - .initial_elements = 65536 / sizeof(DICTIONARY_ITEM), - .requested_element_size = sizeof(DICTIONARY_ITEM), -}; +static ARAL *dict_items_aral = NULL; +static ARAL *dict_shared_items_aral = NULL; -static ARAL dict_shared_items_aral = { - .filename = NULL, - .cache_dir = NULL, - .use_mmap = false, - .initial_elements = 65536 / sizeof(DICTIONARY_ITEM_SHARED), - .requested_element_size = sizeof(DICTIONARY_ITEM_SHARED), -}; +void dictionary_static_items_aral_init(void) { + static SPINLOCK spinlock; + + if(unlikely(!dict_items_aral || !dict_shared_items_aral)) { + netdata_spinlock_lock(&spinlock); + + // we have to check again + if(!dict_items_aral) + dict_items_aral = aral_create( + "dict-items", + sizeof(DICTIONARY_ITEM), + 0, + 4096, + NULL, NULL, false, false); + + // we have to check again + if(!dict_shared_items_aral) + dict_shared_items_aral = aral_create( + "dict-shared-items", + sizeof(DICTIONARY_ITEM_SHARED), + 0, + 4096, + NULL, NULL, false, false); + + netdata_spinlock_unlock(&spinlock); + } +} static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) { DICTIONARY_ITEM *item; size_t size = sizeof(DICTIONARY_ITEM); // item = callocz(1, size); - item = arrayalloc_mallocz(&dict_items_aral); + item = aral_mallocz(dict_items_aral); memset(item, 0, sizeof(DICTIONARY_ITEM)); #ifdef NETDATA_INTERNAL_CHECKS @@ -1276,7 +1291,7 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t else { size = sizeof(DICTIONARY_ITEM_SHARED); // item->shared = callocz(1, size); - item->shared = arrayalloc_mallocz(&dict_shared_items_aral); + item->shared = aral_mallocz(dict_shared_items_aral); memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED)); item->shared->links = 1; @@ -1418,13 +1433,13 @@ static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) value_size += item->shared->value_len; // freez(item->shared); - arrayalloc_freez(&dict_shared_items_aral, item->shared); + aral_freez(dict_shared_items_aral, item->shared); item->shared = NULL; item_size += sizeof(DICTIONARY_ITEM_SHARED); } // freez(item); - arrayalloc_freez(&dict_items_aral, item); + aral_freez(dict_items_aral, item); item_size += sizeof(DICTIONARY_ITEM); @@ -1971,6 +1986,7 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti dict_size += reference_counter_init(dict); dict_size += hashtable_init_unsafe(dict); + dictionary_static_items_aral_init(); pointer_index_init(dict); DICTIONARY_STATS_PLUS_MEMORY(dict, 0, dict_size, 0); -- cgit v1.2.3