diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-01-30 20:36:16 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-30 20:36:16 +0200 |
commit | 7f8f11eb373dfc7bf6ac5a03e57a1b03487a279e (patch) | |
tree | a79f74e904690145b2c6807ca512eec1dd2160ed /libnetdata/dictionary/dictionary.c | |
parent | fd7f39a74426d16f01559bd5aac1a6f90baef57f (diff) |
DBENGINE v2 - improvements part 11 (#14337)
* acquiring / releasing interface for metrics
* metrics registry statistics
* cleanup metrics registry by deleting metrics when they dont have retention anymore; do not double copy the data of pages to be flushed
* print the tier in retention summary
* Open files with buffered instead of direct I/O (test)
* added more metrics stats and fixed unittest
* rename writer functions to avoid confusion with refcounting
* do not release a metric that is not acquired
* Revert to use direct I/O on write -- use direct I/O on read as well
* keep track of ARAL overhead and add it to the memory chart
* aral full check via api
* Cleanup
* give names to ARALs and PGCs
* aral improvements
* restore query expansion to the future
* prefer higher resolution tier when switching plans
* added extent read statistics
* smoother joining of tiers at query engine
* fine tune aral max allocation size
* aral restructuring to hide its internals from the rest of netdata
* aral restructuring; addtion of defrag option to aral to keep the linked list sorted - enabled by default to test it
* fully async aral
* some statistics and cleanup
* fix infinite loop while calculating retention
* aral docs and defragmenting disabled by default
* fix bug and add optimization when defragmenter is not enabled
* aral stress test
* aral speed report and documentation
* added internal checks that all pages are full
* improve internal log about metrics deletion
* metrics registry uses one aral per partition
* metrics registry aral max size to 512 elements per page
* remove data_structures/README.md dependency
---------
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Diffstat (limited to 'libnetdata/dictionary/dictionary.c')
-rw-r--r-- | libnetdata/dictionary/dictionary.c | 60 |
1 files changed, 38 insertions, 22 deletions
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index acc509e7ce..cbe6d0368e 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -789,7 +789,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) { // we didn't get a reference if(item_is_not_referenced_and_can_be_removed(dict, item)) { - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next); dict_item_free_with_hooks(dict, item); deleted++; @@ -1167,9 +1167,9 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); if(dict->options & DICT_OPTION_ADD_IN_FRONT) - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(dict->items.list, item, prev, next); else - DOUBLE_LINKED_LIST_APPEND_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS item->ll_adder_pid = gettid(); @@ -1186,7 +1186,7 @@ static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) { ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(dict->items.list, item, prev, next); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next); #ifdef NETDATA_INTERNAL_CHECKS item->ll_remover_pid = gettid(); @@ -1234,28 +1234,43 @@ static inline size_t item_get_name_len(const DICTIONARY_ITEM *item) { return strlen(item->caller_name); } -static ARAL dict_items_aral = { - .filename = NULL, - .cache_dir = NULL, - .use_mmap = false, - .initial_elements = 65536 / sizeof(DICTIONARY_ITEM), - .requested_element_size = sizeof(DICTIONARY_ITEM), -}; +static ARAL *dict_items_aral = NULL; +static ARAL *dict_shared_items_aral = NULL; -static ARAL dict_shared_items_aral = { - .filename = NULL, - .cache_dir = NULL, - .use_mmap = false, - .initial_elements = 65536 / sizeof(DICTIONARY_ITEM_SHARED), - .requested_element_size = sizeof(DICTIONARY_ITEM_SHARED), -}; +void dictionary_static_items_aral_init(void) { + static SPINLOCK spinlock; + + if(unlikely(!dict_items_aral || !dict_shared_items_aral)) { + netdata_spinlock_lock(&spinlock); + + // we have to check again + if(!dict_items_aral) + dict_items_aral = aral_create( + "dict-items", + sizeof(DICTIONARY_ITEM), + 0, + 4096, + NULL, NULL, false, false); + + // we have to check again + if(!dict_shared_items_aral) + dict_shared_items_aral = aral_create( + "dict-shared-items", + sizeof(DICTIONARY_ITEM_SHARED), + 0, + 4096, + NULL, NULL, false, false); + + netdata_spinlock_unlock(&spinlock); + } +} static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) { DICTIONARY_ITEM *item; size_t size = sizeof(DICTIONARY_ITEM); // item = callocz(1, size); - item = arrayalloc_mallocz(&dict_items_aral); + item = aral_mallocz(dict_items_aral); memset(item, 0, sizeof(DICTIONARY_ITEM)); #ifdef NETDATA_INTERNAL_CHECKS @@ -1276,7 +1291,7 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t else { size = sizeof(DICTIONARY_ITEM_SHARED); // item->shared = callocz(1, size); - item->shared = arrayalloc_mallocz(&dict_shared_items_aral); + item->shared = aral_mallocz(dict_shared_items_aral); memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED)); item->shared->links = 1; @@ -1418,13 +1433,13 @@ static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) value_size += item->shared->value_len; // freez(item->shared); - arrayalloc_freez(&dict_shared_items_aral, item->shared); + aral_freez(dict_shared_items_aral, item->shared); item->shared = NULL; item_size += sizeof(DICTIONARY_ITEM_SHARED); } // freez(item); - arrayalloc_freez(&dict_items_aral, item); + aral_freez(dict_items_aral, item); item_size += sizeof(DICTIONARY_ITEM); @@ -1971,6 +1986,7 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti dict_size += reference_counter_init(dict); dict_size += hashtable_init_unsafe(dict); + dictionary_static_items_aral_init(); pointer_index_init(dict); DICTIONARY_STATS_PLUS_MEMORY(dict, 0, dict_size, 0); |