diff options
42 files changed, 2070 insertions, 1096 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 553b3656aa..386e016acf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -410,6 +410,8 @@ set(LIBNETDATA_FILES libnetdata/string/utf8.h libnetdata/socket/security.c libnetdata/socket/security.h + libnetdata/worker_utilization/worker_utilization.c + libnetdata/worker_utilization/worker_utilization.h libnetdata/circular_buffer/circular_buffer.c libnetdata/circular_buffer/circular_buffer.h) diff --git a/Makefile.am b/Makefile.am index 6dc5f204c3..4a627c25d2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -187,6 +187,8 @@ LIBNETDATA_FILES = \ libnetdata/health/health.c \ libnetdata/health/health.h \ libnetdata/string/utf8.h \ + libnetdata/worker_utilization/worker_utilization.c \ + libnetdata/worker_utilization/worker_utilization.h \ $(NULL) if ENABLE_PLUGIN_EBPF diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 058f6596a6..bd3b7a572c 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -351,6 +351,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que { for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) { if (aclk_query_handlers[i].type == query->type) { + worker_is_busy(i); + debug(D_ACLK, "Processing Queued Message of type: \"%s\"", aclk_query_handlers[i].name); aclk_query_handlers[i].fnc(query_thr, query); if (aclk_stats_enabled) { @@ -361,6 +363,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que ACLK_STATS_UNLOCK; } aclk_query_free(query); + + worker_is_idle(); return; } } @@ -378,21 +382,33 @@ int aclk_query_process_msgs(struct aclk_query_thread *query_thr) return 0; } +static void worker_aclk_register(void) { + worker_register("ACLKQUERY"); + for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) { + worker_register_job_name(i, aclk_query_handlers[i].name); + } +} + /** * Main query processing thread */ void *aclk_query_main_thread(void *ptr) { + worker_aclk_register(); + struct aclk_query_thread *query_thr = ptr; while (!netdata_exit) { aclk_query_process_msgs(query_thr); + worker_is_idle(); QUERY_THREAD_LOCK; if (unlikely(pthread_cond_wait(&query_cond_wait, &query_lock_wait))) sleep_usec(USEC_PER_SEC * 1); QUERY_THREAD_UNLOCK; } + + worker_unregister(); return NULL; } diff --git a/collectors/all.h b/collectors/all.h index 61f3c01bff..3d7304dd55 100644 --- a/collectors/all.h +++ b/collectors/all.h @@ -360,10 +360,8 @@ #define NETDATA_CHART_PRIO_CHECKS 99999 -#define NETDATA_CHART_PRIO_NETDATA_DISKSPACE 132020 #define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030 -#define NETDATA_CHART_PRIO_NETDATA_TC_CPU 135000 -#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 135001 +#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100 #endif //NETDATA_ALL_H diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c index 6924b2bf4e..b4853d3545 100644 --- a/collectors/apps.plugin/apps_plugin.c +++ b/collectors/apps.plugin/apps_plugin.c @@ -4124,6 +4124,8 @@ static int check_capabilities() { int main(int argc, char **argv) { // debug_flags = D_PROCFILE; + clocks_init(); + pagesize = (size_t)sysconf(_SC_PAGESIZE); // set the name for logging diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c index 9453d1b712..bf78624f99 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -2646,11 +2646,26 @@ static inline void discovery_process_cgroup(struct cgroup *cg) { read_cgroup_network_interfaces(cg); } +#define WORKER_DISCOVERY_INIT 0 +#define WORKER_DISCOVERY_FIND 1 +#define WORKER_DISCOVERY_PROCESS 2 +#define WORKER_DISCOVERY_UPDATE 3 +#define WORKER_DISCOVERY_CLEANUP 4 +#define WORKER_DISCOVERY_COPY 5 +#define WORKER_DISCOVERY_SHARE 6 +#define WORKER_DISCOVERY_LOCK 7 + +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8 +#endif + static inline void discovery_find_all_cgroups() { debug(D_CGROUP, "searching for cgroups"); + worker_is_busy(WORKER_DISCOVERY_INIT); discovery_mark_all_cgroups_as_unavailable(); + worker_is_busy(WORKER_DISCOVERY_FIND); if (!cgroup_use_unified_cgroups) { discovery_find_all_cgroups_v1(); } else { @@ -2659,16 +2674,25 @@ static inline void discovery_find_all_cgroups() { struct cgroup *cg; for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) { + worker_is_busy(WORKER_DISCOVERY_PROCESS); discovery_process_cgroup(cg); } + worker_is_busy(WORKER_DISCOVERY_UPDATE); discovery_update_filenames(); + worker_is_busy(WORKER_DISCOVERY_LOCK); uv_mutex_lock(&cgroup_root_mutex); + + worker_is_busy(WORKER_DISCOVERY_CLEANUP); discovery_cleanup_all_cgroups(); + + worker_is_busy(WORKER_DISCOVERY_COPY); discovery_copy_discovered_cgroups_to_reader(); + uv_mutex_unlock(&cgroup_root_mutex); + worker_is_busy(WORKER_DISCOVERY_SHARE); discovery_share_cgroups_with_ebpf(); debug(D_CGROUP, "done searching for cgroups"); @@ -2678,7 +2702,19 @@ void cgroup_discovery_worker(void *ptr) { UNUSED(ptr); + worker_register("CGROUPSDISC"); + worker_register_job_name(WORKER_DISCOVERY_INIT, "init"); + worker_register_job_name(WORKER_DISCOVERY_FIND, "find"); + worker_register_job_name(WORKER_DISCOVERY_PROCESS, "process"); + worker_register_job_name(WORKER_DISCOVERY_UPDATE, "update"); + worker_register_job_name(WORKER_DISCOVERY_CLEANUP, "cleanup"); + worker_register_job_name(WORKER_DISCOVERY_COPY, "copy"); + worker_register_job_name(WORKER_DISCOVERY_SHARE, "share"); + worker_register_job_name(WORKER_DISCOVERY_LOCK, "lock"); + while (!netdata_exit) { + worker_is_idle(); + uv_mutex_lock(&discovery_thread.mutex); while (!discovery_thread.start_discovery) uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex); @@ -2692,6 +2728,7 @@ void cgroup_discovery_worker(void *ptr) } discovery_thread.exited = 1; + worker_unregister(); } // ---------------------------------------------------------------------------- @@ -4650,6 +4687,8 @@ void update_cgroup_charts(int update_every) { // cgroups main static void cgroup_main_cleanup(void *ptr) { + worker_unregister(); + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; @@ -4687,24 +4726,30 @@ static void cgroup_main_cleanup(void *ptr) { static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } +#define WORKER_CGROUPS_LOCK 0 +#define WORKER_CGROUPS_READ 1 +#define WORKER_CGROUPS_CHART 2 + +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3 +#endif + void *cgroups_main(void *ptr) { - netdata_thread_cleanup_push(cgroup_main_cleanup, ptr); + worker_register("CGROUPS"); + worker_register_job_name(WORKER_CGROUPS_LOCK, "lock"); + worker_register_job_name(WORKER_CGROUPS_READ, "read"); + worker_register_job_name(WORKER_CGROUPS_READ, "chart"); - struct rusage thread; + netdata_thread_cleanup_push(cgroup_main_cleanup, ptr); if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) { is_inside_k8s = 1; cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_YES; } - // when ZERO, attempt to do it - int vdo_cpu_netdata = config_get_boolean("plugin:cgroups", "cgroups plugin resource charts", 1); - read_cgroup_plugin_configuration(); netdata_cgroup_ebpf_initialize_shm(); - RRDSET *stcpu_thread = NULL; - if (uv_mutex_init(&cgroup_root_mutex)) { error("CGROUP: cannot initialize mutex for the main cgroup list"); goto exit; @@ -4736,6 +4781,8 @@ void *cgroups_main(void *ptr) { usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0; while(!netdata_exit) { + worker_is_idle(); + usec_t hb_dt = heartbeat_next(&hb, step); if(unlikely(netdata_exit)) break; @@ -4747,46 +4794,21 @@ void *cgroups_main(void *ptr) { cgroups_check = 0; } + worker_is_busy(WORKER_CGROUPS_LOCK); uv_mutex_lock(&cgroup_root_mutex); - read_all_discovered_cgroups(cgroup_root); - update_cgroup_charts(cgroup_update_every); - uv_mutex_unlock(&cgroup_root_mutex); - - // -------------------------------------------------------------------- - - if(vdo_cpu_netdata) { - getrusage(RUSAGE_THREAD, &thread); - if(unlikely(!stcpu_thread)) { - - stcpu_thread = rrdset_create_localhost( - "netdata" - , "plugin_cgroups_cpu" - , NULL - , "cgroups" - , NULL - , "Netdata CGroups Plugin CPU usage" - , "milliseconds/s" - , PLUGIN_CGROUPS_NAME - , "stats" - , 132000 - , cgroup_update_every - , RRDSET_TYPE_STACKED - ); + worker_is_busy(WORKER_CGROUPS_READ); + read_all_discovered_cgroups(cgroup_root); - rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - } - else - rrdset_next(stcpu_thread); + worker_is_busy(WORKER_CGROUPS_CHART); + update_cgroup_charts(cgroup_update_every); - rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); - rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); - rrdset_done(stcpu_thread); - } + worker_is_idle(); + uv_mutex_unlock(&cgroup_root_mutex); } exit: + worker_unregister(); netdata_thread_cleanup_pop(1); return NULL; } diff --git a/collectors/cups.plugin/cups_plugin.c b/collectors/cups.plugin/cups_plugin.c index cc57dbf1f5..46bbc19bb8 100644 --- a/collectors/cups.plugin/cups_plugin.c +++ b/collectors/cups.plugin/cups_plugin.c @@ -224,6 +224,7 @@ void reset_metrics() { } int main(int argc, char **argv) { + clocks_init(); // ------------------------------------------------------------------------ // initialization of netdata plugin diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index b6a52c0611..13806277c3 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -365,6 +365,8 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { } static void diskspace_main_cleanup(void *ptr) { + worker_unregister(); + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; @@ -373,10 +375,21 @@ static void diskspace_main_cleanup(void *ptr) { static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } +#define WORKER_JOB_MOUNTINFO 0 +#define WORKER_JOB_MOUNTPOINT 1 +#define WORKER_JOB_CLEANUP 2 + +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3 +#endif + void *diskspace_main(void *ptr) { - netdata_thread_cleanup_push(diskspace_main_cleanup, ptr); + worker_register("DISKSPACE"); + worker_register_job_name(WORKER_JOB_MOUNTINFO, "mountinfo"); + worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint"); + worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup"); - int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1); + netdata_thread_cleanup_push(diskspace_main_cleanup, ptr); cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points); @@ -388,14 +401,11 @@ void *diskspace_main(void *ptr) { if(check_for_new_mountpoints_every < update_every) check_for_new_mountpoints_every = update_every; - struct rusage thread; - - usec_t duration = 0; usec_t step = update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); while(!netdata_exit) { - duration = heartbeat_monotonic_dt_to_now_usec(&hb); + worker_is_idle(); /* usec_t hb_dt = */ heartbeat_next(&hb, step); if(unlikely(netdata_exit)) break; @@ -404,9 +414,9 @@ void *diskspace_main(void *ptr) { // -------------------------------------------------------------------------- // this is smart enough not to reload it every time + worker_is_busy(WORKER_JOB_MOUNTINFO); mountinfo_reload(0); - // -------------------------------------------------------------------------- // disk space metrics @@ -420,80 +430,20 @@ void *diskspace_main(void *ptr) { if(mi->flags & MOUNTINFO_READONLY && !strcmp(mi->root, mi->mount_point)) continue; + worker_is_busy(WORKER_JOB_MOUNTPOINT); do_disk_space_stats(mi, update_every); if(unlikely(netdata_exit)) break; } if(unlikely(netdata_exit)) break; - if(dict_mountpoints) + if(dict_mountpoints) { + worker_is_busy(WORKER_JOB_CLEANUP); dictionary_get_all(dict_mountpoints, mount_point_cleanup, NULL); - - if(vdo_cpu_netdata) { - static RRDSET *stcpu_thread = NULL, *st_duration = NULL; - static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL; - - // ---------------------------------------------------------------- - - getrusage(RUSAGE_THREAD, &thread); - - if(unlikely(!stcpu_thread)) { - stcpu_thread = rrdset_create_localhost( - "netdata" - , "plugin_diskspace" - , NULL - , "diskspace" - , NULL - , "Netdata Disk Space Plugin CPU usage" - , "milliseconds/s" - , PLUGIN_DISKSPACE_NAME - , NULL - , NETDATA_CHART_PRIO_NETDATA_DISKSPACE - , update_every - , RRDSET_TYPE_STACKED - ); - - rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - } - else - rrdset_next(stcpu_thread); - - rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); - rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); - rrdset_done(stcpu_thread); - - // ---------------------------------------------------------------- - - if(unlikely(!st_duration)) { - st_duration = rrdset_create_localhost( - "netdata" - , "plugin_diskspace_dt" - , NULL - , "diskspace" - , NULL - , "Netdata Disk Space Plugin Duration" - , "milliseconds/run" - , PLUGIN_DISKSPACE_NAME - , NULL - , 132021 - , update_every - , RRDSET_TYPE_AREA - ); - - rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - } - else - rrdset_next(st_duration); - - rrddim_set_by_pointer(st_duration, rd_duration, duration); - rrdset_done(st_duration); - - // ---------------------------------------------------------------- - - if(unlikely(netdata_exit)) break; } + } + worker_unregister(); netdata_thread_cleanup_pop(1); return NULL; diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c index eb9332273e..9f411bb8bf 100644 --- a/collectors/ebpf.plugin/ebpf.c +++ b/collectors/ebpf.plugin/ebpf.c @@ -1864,6 +1864,8 @@ static void ebpf_manage_pid(pid_t pid) */ int main(int argc, char **argv) { + clocks_init(); + set_global_variables(); ebpf_parse_args(argc, argv); ebpf_manage_pid(getpid()); diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c index 97ca1d9ae0..a52ece3f94 100644 --- a/collectors/freebsd.plugin/plugin_freebsd.c +++ b/collectors/freebsd.plugin/plugin_freebsd.c @@ -9,7 +9,6 @@ static struct freebsd_module { int enabled; int (*func)(int update_every, usec_t dt); - usec_t duration; RRDDIM *rd; @@ -68,8 +67,14 @@ static struct freebsd_module { {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL} }; +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 33 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 33 +#endif + static void freebsd_main_cleanup(void *ptr) { + worker_unregister(); + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; @@ -80,9 +85,9 @@ static void freebsd_main_cleanup(void *ptr) void *freebsd_main(void *ptr) { - netdata_thread_cleanup_push(freebsd_main_cleanup, ptr); + worker_register("FREEBSD"); - int vdo_cpu_netdata = config_get_boolean("plugin:freebsd", "netdata server resources", 1); + netdata_thread_cleanup_push(freebsd_main_cleanup, ptr); // initialize FreeBSD plugin if (freebsd_plugin_init()) @@ -94,8 +99,9 @@ void *freebsd_main(void *ptr) struct freebsd_module *pm = &freebsd_modules[i]; pm->enabled = config_get_boolean("plugin:freebsd", pm->name, pm->enabled); - pm->duration = 0ULL; pm->rd = NULL; + + worker_register_job_name(i, freebsd_modules[i].dim); } usec_t step = localhost->rrd_update_every * USEC_PER_SEC; @@ -103,14 +109,13 @@ void *freebsd_main(void *ptr) heartbeat_init(&hb); while (!netdata_exit) { + worker_is_idle(); + usec_t hb_dt = heartbeat_next(&hb, step); - usec_t duration = 0ULL; if (unlikely(netdata_exit)) break; - // BEGIN -- the job to be done - for (i = 0; freebsd_modules[i].name; i++) { struct freebsd_module *pm = &freebsd_modules[i]; if (unlikely(!pm->enabled)) @@ -118,92 +123,12 @@ void *freebsd_main(void *ptr) debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name); + worker_is_busy(i); pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); - pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration; - duration += pm->duration; if (unlikely(netdata_exit)) break; } - - // END -- the job is done - - if (vdo_cpu_netdata) { - static RRDSET *st_cpu_thread = NULL, *st_duration = NULL; - static RRDDIM *rd_user = NULL, *rd_system = NULL; - - // ---------------------------------------------------------------- - - struct rusage thread; - getrusage(RUSAGE_THREAD, &thread); - - if (unlikely(!st_cpu_thread)) { - st_cpu_thread = rrdset_create_localhost( - "netdata", - "plugin_freebsd_cpu", - NULL, - "freebsd", - NULL, - "Netdata FreeBSD plugin CPU usage", - "milliseconds/s", - "freebsd.plugin", - "stats", - 132000, - localhost->rrd_update_every, - RRDSET_TYPE_STACKED); - - rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL); - rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL); - } else { - rrdset_next(st_cpu_thread); - } - - rrddim_set_by_pointer( - st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec); - rrddim_set_by_pointer( - st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec); - rrdset_done(st_cpu_thread); - - // ---------------------------------------------------------------- - - if (unlikely(!st_duration)) { - st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules"); - - if (!st_duration) { - st_duration = rrdset_create_localhost( - "netdata", - "plugin_freebsd_modules", - NULL, - "freebsd", - NULL, - "Netdata FreeBSD plugin modules durations", - "milliseconds/run", - "freebsd.plugin", - "stats", - 132001, - localhost->rrd_update_every, - RRDSET_TYPE_STACKED); - - for (i = 0; freebsd_modules[i].name; i++) { - struct freebsd_module *pm = &freebsd_modules[i]; - if (unlikely(!pm->enabled)) - continue; - - pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - } - } - } else - rrdset_next(st_duration); - - for (i = 0; freebsd_modules[i].name; i++) { - struct freebsd_module *pm = &freebsd_modules[i]; - if (unlikely(!pm->enabled)) - continue; - - rrddim_set_by_pointer(st_duration, pm->rd, pm->duration); - } - rrdset_done(st_duration); - } } netdata_thread_cleanup_pop(1); diff --git a/collectors/freeipmi.plugin/freeipmi_plugin.c b/collectors/freeipmi.plugin/freeipmi_plugin.c index 6c6f3d747f..351b6e32be 100644 --- a/collectors/freeipmi.plugin/freeipmi_plugin.c +++ b/collectors/freeipmi.plugin/freeipmi_plugin.c |