DBENGINE v2 - improvements part 12 (#14379)

* parallel initialization of tiers

* do not spawn multiple dbengine event loops

* user configurable dbengine parallel initialization

* size netdata based on the real cpu cores available on the system netdata runs, not on the system monitored

* user configurable system cpus

* move cpuset parsing to os.c/.h

* fix replication of misaligned chart dimensions

* give a different path to each tier thread

* statically allocate the path into the initialization structure

* use aral for reusing dbengine pages

* dictionaries uses ARAL for fixed sized values

* fix compilation without internal checks

* journal v2 index uses aral

* test to see judy allocations

* judy allocations using aral

* Add config option to select if dbengine will use direct I/O (default is yes)

* V1 journafiles will use uv_fs_read instead of mmap (respect the direct I/O setting)

* Remove sqlite3IsMemdb as it is unused

* Fix compilation error when --disable-dbengine is used

* use aral for dbengine work_cmds

* changed aral API to support new features

* pgc and mrg aral overheads

* rrdeng opcodes using aral

* better structuring and naming

* dbegnine query handles using aral

* page descriptors using aral

* remove obsolete linking

* extent io descriptors using aral

* aral keeps one last page alive

* add missing return value

* added judy aral overhead

* pdc now uses aral

* page_details now use aral

* epdl and deol using aral - make sure ARALs are initialized before spawning the event loop

* remove unused linking

* pgc now uses one aral per partition

* aral measure maximum allocation queue

* aral to allocate pages in parallel

* aral parallel pages allocation when needed

* aral cleanup

* track page allocation and page population separately

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

8 files changed, 21 insertions, 58 deletions

diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index 9f9178c367..d312f44300 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -3516,52 +3516,15 @@ static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len)
     return buffer;
 }
 
-static inline unsigned long long cpuset_str2ull(char **s) {
-    unsigned long long n = 0;
-    char c;
-    for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
-        n *= 10;
-        n += c - '0';
-    }
-    return n;
-}
-
 static inline void update_cpu_limits(char **filename, unsigned long long *value, struct cgroup *cg) {
     if(*filename) {
         int ret = -1;
 
         if(value == &cg->cpuset_cpus) {
-            static char *buf = NULL;
-            static size_t buf_size = 0;
-
-            if(!buf) {
-                buf_size = 100U + 6 * get_system_cpus(); // taken from kernel/cgroup/cpuset.c
-                buf = mallocz(buf_size + 1);
-            }
-
-            ret = read_file(*filename, buf, buf_size);
-
-            if(!ret) {
-                char *s = buf;
-                unsigned long long ncpus = 0;
-
-                // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
-                while(*s) {
-                    unsigned long long n = cpuset_str2ull(&s);
-                    ncpus++;
-                    if(*s == ',') {
-                        s++;
-                        continue;
-                    }
-                    if(*s == '-') {
-                        s++;
-                        unsigned long long m = cpuset_str2ull(&s);
-                        ncpus += m - n; // calculate the number of cpus in the region
-                    }
-                    s++;
-                }
-
-                if(likely(ncpus)) *value = ncpus;
+            unsigned long ncpus = read_cpuset_cpus(*filename, get_system_cpus());
+            if(ncpus) {
+                *value = ncpus;
+                ret = 0;
             }
         }
         else if(value == &cg->cpu_cfs_period) {
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index 96622c4aee..743612ffb1 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -319,7 +319,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                 , SIMPLE_PATTERN_EXACT
         );
 
-        dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors);
+        dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors, 0);
     }
 
     struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
index 48acfd7dc1..f172ccc262 100644
--- a/collectors/plugins.d/pluginsd_parser.c
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -525,7 +525,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may
 }
 
 void inflight_functions_init(PARSER *parser) {
-    parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions);
+    parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0);
     dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
     dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
     dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);
diff --git a/collectors/proc.plugin/proc_self_mountinfo.c b/collectors/proc.plugin/proc_self_mountinfo.c
index 0120310ae5..0483749c37 100644
--- a/collectors/proc.plugin/proc_self_mountinfo.c
+++ b/collectors/proc.plugin/proc_self_mountinfo.c
@@ -229,7 +229,7 @@ struct mountinfo *mountinfo_read(int do_statvfs) {
     // create a dictionary to track uniqueness
     DICTIONARY *dict = dictionary_create_advanced(
             DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE,
-            &dictionary_stats_category_collectors);
+            &dictionary_stats_category_collectors, 0);
 
     unsigned long l, lines = procfile_lines(ff);
     for(l = 0; l < lines ;l++) {
diff --git a/collectors/proc.plugin/proc_spl_kstat_zfs.c b/collectors/proc.plugin/proc_spl_kstat_zfs.c
index 3a1c9160c7..0db9970c36 100644
--- a/collectors/proc.plugin/proc_spl_kstat_zfs.c
+++ b/collectors/proc.plugin/proc_spl_kstat_zfs.c
@@ -322,7 +322,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
         snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
         dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename);
 
-        zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
+        zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
 
         do_zfs_pool_state = 1;
     }
diff --git a/collectors/proc.plugin/sys_block_zram.c b/collectors/proc.plugin/sys_block_zram.c
index b95786a3fd..1be725b103 100644
--- a/collectors/proc.plugin/sys_block_zram.c
+++ b/collectors/proc.plugin/sys_block_zram.c
@@ -267,7 +267,7 @@ int do_sys_block_zram(int update_every, usec_t dt) {
         }
         procfile_close(ff);
 
-        devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
+        devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
         device_count = init_devices(devices, (unsigned int)zram_id, update_every);
     }
 
diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c
index c359a2085a..d15129b9c7 100644
--- a/collectors/statsd.plugin/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -595,7 +595,7 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
     }
 
     if (unlikely(!m->set.dict)) {
-        m->set.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
+        m->set.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
         dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
         m->set.unique = 0;
     }
@@ -635,7 +635,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
         statsd_reset_metric(m);
 
     if (unlikely(!m->dictionary.dict)) {
-        m->dictionary.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
+        m->dictionary.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
         dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
         m->dictionary.unique = 0;
     }
@@ -1337,7 +1337,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
             else if(app) {
                 if(!strcmp(s, "dictionary")) {
                     if(!app->dict)
-                        app->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
+                        app->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
 
                     dict = app->dict;
                 }
@@ -2422,13 +2422,13 @@ void *statsd_main(void *ptr) {
 
     netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
 
-    statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.histograms.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.dictionaries.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.sets.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
-    statsd.timers.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors);
+    statsd.gauges.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.meters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.counters.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.histograms.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.dictionaries.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.sets.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
+    statsd.timers.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
 
     dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges);
     dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters);
diff --git a/collectors/tc.plugin/plugin_tc.c b/collectors/tc.plugin/plugin_tc.c
index 59f0302886..a8ceca4494 100644
--- a/collectors/tc.plugin/plugin_tc.c
+++ b/collectors/tc.plugin/plugin_tc.c
@@ -98,7 +98,7 @@ static bool tc_class_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse
 
 static void tc_class_index_init(struct tc_device *d) {
     if(!d->classes) {
-        d->classes = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
+        d->classes = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors, 0);
 
         dictionary_register_delete_callback(d->classes, tc_class_free_callback, d);
         dictionary_register_conflict_callback(d->classes, tc_class_conflict_callback, d);
@@ -146,7 +146,7 @@ static void tc_device_index_init() {
     if(!tc_device_root_index) {
         tc_device_root_index = dictionary_create_advanced(
             DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED | DICT_OPTION_ADD_IN_FRONT,
-            &dictionary_stats_category_collectors);
+            &dictionary_stats_category_collectors, 0);
 
         dictionary_register_insert_callback(tc_device_root_index, tc_device_add_callback, NULL);
         dictionary_register_delete_callback(tc_device_root_index, tc_device_free_callback, NULL);