Workers utilization charts (#12807)

* initial version of worker utilization

* working example

* without mutexes

* monitoring DBENGINE, ACLKSYNC, WEB workers

* added charts to monitor worker usage

* fixed charts units

* updated contexts

* updated priorities

* added documentation

* converted threads to stacked chart

* One query per query thread

* Revert "One query per query thread"

This reverts commit 6aeb391f5987c3c6ba2864b559fd7f0cd64b14d3.

* fixed priority for web charts

* read worker cpu utilization from proc

* read workers cpu utilization via /proc/self/task/PID/stat, so that we have cpu utilization even when the jobs are too long to finish within our update_every frequency

* disabled web server cpu utilization monitoring - it is now monitored by worker utilization

* tight integration of worker utilization to web server

* monitoring statsd worker threads

* code cleanup and renaming of variables

* contrained worker and statistics conflict to just one variable

* support for rendering jobs per type

* better priorities and removed the total jobs chart

* added busy time in ms per job type

* added proc.plugin monitoring, switch clock to MONOTONIC_RAW if available, global statistics now cleans up old worker threads

* isolated worker thread families

* added cgroups.plugin workers

* remove unneeded dimensions when then expected worker is just one

* plugins.d and streaming monitoring

* rebased; support worker_is_busy() to be called one after another

* added diskspace plugin monitoring

* added tc.plugin monitoring

* added ML threads monitoring

* dont create dimensions and charts that are not needed

* fix crash when job types are added on the fly

* added timex and idlejitter plugins; collected heartbeat statistics; reworked heartbeat according to the POSIX

* the right name is heartbeat for this chart

* monitor streaming senders

* added streaming senders to global stats

* prevent division by zero

* added clock_init() to external C plugins

* added freebsd and macos plugins

* added freebsd and macos to global statistics

* dont use new as a variable; address compiler warnings on FreeBSD and MacOS

* refactored contexts to be unique; added health threads monitoring

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

42 files changed, 2070 insertions, 1096 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 553b3656aa..386e016acf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -410,6 +410,8 @@ set(LIBNETDATA_FILES
         libnetdata/string/utf8.h
         libnetdata/socket/security.c
         libnetdata/socket/security.h
+        libnetdata/worker_utilization/worker_utilization.c
+        libnetdata/worker_utilization/worker_utilization.h
         libnetdata/circular_buffer/circular_buffer.c
         libnetdata/circular_buffer/circular_buffer.h)
 
diff --git a/Makefile.am b/Makefile.am
index 6dc5f204c3..4a627c25d2 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -187,6 +187,8 @@ LIBNETDATA_FILES = \
     libnetdata/health/health.c \
     libnetdata/health/health.h \
     libnetdata/string/utf8.h \
+    libnetdata/worker_utilization/worker_utilization.c \
+    libnetdata/worker_utilization/worker_utilization.h \
     $(NULL)
 
 if ENABLE_PLUGIN_EBPF
diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c
index 058f6596a6..bd3b7a572c 100644
--- a/aclk/aclk_query.c
+++ b/aclk/aclk_query.c
@@ -351,6 +351,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que
 {
     for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) {
         if (aclk_query_handlers[i].type == query->type) {
+            worker_is_busy(i);
+
             debug(D_ACLK, "Processing Queued Message of type: \"%s\"", aclk_query_handlers[i].name);
             aclk_query_handlers[i].fnc(query_thr, query);
             if (aclk_stats_enabled) {
@@ -361,6 +363,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que
                 ACLK_STATS_UNLOCK;
             }
             aclk_query_free(query);
+
+            worker_is_idle();
             return;
         }
     }
@@ -378,21 +382,33 @@ int aclk_query_process_msgs(struct aclk_query_thread *query_thr)
     return 0;
 }
 
+static void worker_aclk_register(void) {
+    worker_register("ACLKQUERY");
+    for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) {
+        worker_register_job_name(i, aclk_query_handlers[i].name);
+    }
+}
+
 /**
  * Main query processing thread
  */
 void *aclk_query_main_thread(void *ptr)
 {
+    worker_aclk_register();
+
     struct aclk_query_thread *query_thr = ptr;
 
     while (!netdata_exit) {
         aclk_query_process_msgs(query_thr);
 
+        worker_is_idle();
         QUERY_THREAD_LOCK;
         if (unlikely(pthread_cond_wait(&query_cond_wait, &query_lock_wait)))
             sleep_usec(USEC_PER_SEC * 1);
         QUERY_THREAD_UNLOCK;
     }
+
+    worker_unregister();
     return NULL;
 }
 
diff --git a/collectors/all.h b/collectors/all.h
index 61f3c01bff..3d7304dd55 100644
--- a/collectors/all.h
+++ b/collectors/all.h
@@ -360,10 +360,8 @@
 
 #define NETDATA_CHART_PRIO_CHECKS                    99999
 
-#define NETDATA_CHART_PRIO_NETDATA_DISKSPACE        132020
 #define NETDATA_CHART_PRIO_NETDATA_TIMEX            132030
-#define NETDATA_CHART_PRIO_NETDATA_TC_CPU           135000
-#define NETDATA_CHART_PRIO_NETDATA_TC_TIME          135001
+#define NETDATA_CHART_PRIO_NETDATA_TC_TIME          1000100
 
 
 #endif //NETDATA_ALL_H
diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c
index 6924b2bf4e..b4853d3545 100644
--- a/collectors/apps.plugin/apps_plugin.c
+++ b/collectors/apps.plugin/apps_plugin.c
@@ -4124,6 +4124,8 @@ static int check_capabilities() {
 int main(int argc, char **argv) {
     // debug_flags = D_PROCFILE;
 
+    clocks_init();
+
     pagesize = (size_t)sysconf(_SC_PAGESIZE);
 
     // set the name for logging
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index 9453d1b712..bf78624f99 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -2646,11 +2646,26 @@ static inline void discovery_process_cgroup(struct cgroup *cg) {
     read_cgroup_network_interfaces(cg);
 }
 
+#define WORKER_DISCOVERY_INIT 0
+#define WORKER_DISCOVERY_FIND 1
+#define WORKER_DISCOVERY_PROCESS 2
+#define WORKER_DISCOVERY_UPDATE 3
+#define WORKER_DISCOVERY_CLEANUP 4
+#define WORKER_DISCOVERY_COPY 5
+#define WORKER_DISCOVERY_SHARE 6
+#define WORKER_DISCOVERY_LOCK 7
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
+#endif
+
 static inline void discovery_find_all_cgroups() {
     debug(D_CGROUP, "searching for cgroups");
 
+    worker_is_busy(WORKER_DISCOVERY_INIT);
     discovery_mark_all_cgroups_as_unavailable();
 
+    worker_is_busy(WORKER_DISCOVERY_FIND);
     if (!cgroup_use_unified_cgroups) {
         discovery_find_all_cgroups_v1();
     } else {
@@ -2659,16 +2674,25 @@ static inline void discovery_find_all_cgroups() {
 
     struct cgroup *cg;
     for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) {
+        worker_is_busy(WORKER_DISCOVERY_PROCESS);
         discovery_process_cgroup(cg);
     }
 
+    worker_is_busy(WORKER_DISCOVERY_UPDATE);
     discovery_update_filenames();
 
+    worker_is_busy(WORKER_DISCOVERY_LOCK);
     uv_mutex_lock(&cgroup_root_mutex);
+
+    worker_is_busy(WORKER_DISCOVERY_CLEANUP);
     discovery_cleanup_all_cgroups();
+
+    worker_is_busy(WORKER_DISCOVERY_COPY);
     discovery_copy_discovered_cgroups_to_reader();
+
     uv_mutex_unlock(&cgroup_root_mutex);
 
+    worker_is_busy(WORKER_DISCOVERY_SHARE);
     discovery_share_cgroups_with_ebpf();
 
     debug(D_CGROUP, "done searching for cgroups");
@@ -2678,7 +2702,19 @@ void cgroup_discovery_worker(void *ptr)
 {
     UNUSED(ptr);
 
+    worker_register("CGROUPSDISC");
+    worker_register_job_name(WORKER_DISCOVERY_INIT, "init");
+    worker_register_job_name(WORKER_DISCOVERY_FIND, "find");
+    worker_register_job_name(WORKER_DISCOVERY_PROCESS, "process");
+    worker_register_job_name(WORKER_DISCOVERY_UPDATE, "update");
+    worker_register_job_name(WORKER_DISCOVERY_CLEANUP, "cleanup");
+    worker_register_job_name(WORKER_DISCOVERY_COPY, "copy");
+    worker_register_job_name(WORKER_DISCOVERY_SHARE, "share");
+    worker_register_job_name(WORKER_DISCOVERY_LOCK, "lock");
+
     while (!netdata_exit) {
+        worker_is_idle();
+
         uv_mutex_lock(&discovery_thread.mutex);
         while (!discovery_thread.start_discovery)
             uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);
@@ -2692,6 +2728,7 @@ void cgroup_discovery_worker(void *ptr)
     }
 
     discovery_thread.exited = 1;
+    worker_unregister();
 } 
 
 // ----------------------------------------------------------------------------
@@ -4650,6 +4687,8 @@ void update_cgroup_charts(int update_every) {
 // cgroups main
 
 static void cgroup_main_cleanup(void *ptr) {
+    worker_unregister();
+
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -4687,24 +4726,30 @@ static void cgroup_main_cleanup(void *ptr) {
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
+#define WORKER_CGROUPS_LOCK 0
+#define WORKER_CGROUPS_READ 1
+#define WORKER_CGROUPS_CHART 2
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
+#endif
+
 void *cgroups_main(void *ptr) {
-    netdata_thread_cleanup_push(cgroup_main_cleanup, ptr);
+    worker_register("CGROUPS");
+    worker_register_job_name(WORKER_CGROUPS_LOCK, "lock");
+    worker_register_job_name(WORKER_CGROUPS_READ, "read");
+    worker_register_job_name(WORKER_CGROUPS_READ, "chart");
 
-    struct rusage thread;
+    netdata_thread_cleanup_push(cgroup_main_cleanup, ptr);
 
     if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) {
         is_inside_k8s = 1;
         cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_YES;
     }
 
-    // when ZERO, attempt to do it
-    int vdo_cpu_netdata = config_get_boolean("plugin:cgroups", "cgroups plugin resource charts", 1);
-
     read_cgroup_plugin_configuration();
     netdata_cgroup_ebpf_initialize_shm();
 
-    RRDSET *stcpu_thread = NULL;
-
     if (uv_mutex_init(&cgroup_root_mutex)) {
         error("CGROUP: cannot initialize mutex for the main cgroup list");
         goto exit;
@@ -4736,6 +4781,8 @@ void *cgroups_main(void *ptr) {
     usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0;
 
     while(!netdata_exit) {
+        worker_is_idle();
+
         usec_t hb_dt = heartbeat_next(&hb, step);
         if(unlikely(netdata_exit)) break;
 
@@ -4747,46 +4794,21 @@ void *cgroups_main(void *ptr) {
             cgroups_check = 0;
         }
 
+        worker_is_busy(WORKER_CGROUPS_LOCK);
         uv_mutex_lock(&cgroup_root_mutex);
-        read_all_discovered_cgroups(cgroup_root);
-        update_cgroup_charts(cgroup_update_every);
-        uv_mutex_unlock(&cgroup_root_mutex);
-
-        // --------------------------------------------------------------------
-
-        if(vdo_cpu_netdata) {
-            getrusage(RUSAGE_THREAD, &thread);
 
-            if(unlikely(!stcpu_thread)) {
-
-                stcpu_thread = rrdset_create_localhost(
-                        "netdata"
-                        , "plugin_cgroups_cpu"
-                        , NULL
-                        , "cgroups"
-                        , NULL
-                        , "Netdata CGroups Plugin CPU usage"
-                        , "milliseconds/s"
-                        , PLUGIN_CGROUPS_NAME
-                        , "stats"
-                        , 132000
-                        , cgroup_update_every
-                        , RRDSET_TYPE_STACKED
-                );
+        worker_is_busy(WORKER_CGROUPS_READ);
+        read_all_discovered_cgroups(cgroup_root);
 
-                rrddim_add(stcpu_thread, "user",  NULL,  1, 1000, RRD_ALGORITHM_INCREMENTAL);
-                rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-            }
-            else
-                rrdset_next(stcpu_thread);
+        worker_is_busy(WORKER_CGROUPS_CHART);
+        update_cgroup_charts(cgroup_update_every);
 
-            rrddim_set(stcpu_thread, "user"  , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
-            rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
-            rrdset_done(stcpu_thread);
-        }
+        worker_is_idle();
+        uv_mutex_unlock(&cgroup_root_mutex);
     }
 
 exit:
+    worker_unregister();
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
diff --git a/collectors/cups.plugin/cups_plugin.c b/collectors/cups.plugin/cups_plugin.c
index cc57dbf1f5..46bbc19bb8 100644
--- a/collectors/cups.plugin/cups_plugin.c
+++ b/collectors/cups.plugin/cups_plugin.c
@@ -224,6 +224,7 @@ void reset_metrics() {
 }
 
 int main(int argc, char **argv) {
+    clocks_init();
 
     // ------------------------------------------------------------------------
     // initialization of netdata plugin
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index b6a52c0611..13806277c3 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -365,6 +365,8 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
 }
 
 static void diskspace_main_cleanup(void *ptr) {
+    worker_unregister();
+
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -373,10 +375,21 @@ static void diskspace_main_cleanup(void *ptr) {
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
+#define WORKER_JOB_MOUNTINFO 0
+#define WORKER_JOB_MOUNTPOINT 1
+#define WORKER_JOB_CLEANUP 2
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
+#endif
+
 void *diskspace_main(void *ptr) {
-    netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
+    worker_register("DISKSPACE");
+    worker_register_job_name(WORKER_JOB_MOUNTINFO, "mountinfo");
+    worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint");
+    worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup");
 
-    int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1);
+    netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
 
     cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
 
@@ -388,14 +401,11 @@ void *diskspace_main(void *ptr) {
     if(check_for_new_mountpoints_every < update_every)
         check_for_new_mountpoints_every = update_every;
 
-    struct rusage thread;
-
-    usec_t duration = 0;
     usec_t step = update_every * USEC_PER_SEC;
     heartbeat_t hb;
     heartbeat_init(&hb);
     while(!netdata_exit) {
-        duration = heartbeat_monotonic_dt_to_now_usec(&hb);
+        worker_is_idle();
         /* usec_t hb_dt = */ heartbeat_next(&hb, step);
 
         if(unlikely(netdata_exit)) break;
@@ -404,9 +414,9 @@ void *diskspace_main(void *ptr) {
         // --------------------------------------------------------------------------
         // this is smart enough not to reload it every time
 
+        worker_is_busy(WORKER_JOB_MOUNTINFO);
         mountinfo_reload(0);
 
-
         // --------------------------------------------------------------------------
         // disk space metrics
 
@@ -420,80 +430,20 @@ void *diskspace_main(void *ptr) {
             if(mi->flags & MOUNTINFO_READONLY && !strcmp(mi->root, mi->mount_point))
                 continue;
 
+            worker_is_busy(WORKER_JOB_MOUNTPOINT);
             do_disk_space_stats(mi, update_every);
             if(unlikely(netdata_exit)) break;
         }
 
         if(unlikely(netdata_exit)) break;
 
-        if(dict_mountpoints)
+        if(dict_mountpoints) {
+            worker_is_busy(WORKER_JOB_CLEANUP);
             dictionary_get_all(dict_mountpoints, mount_point_cleanup, NULL);
-
-        if(vdo_cpu_netdata) {
-            static RRDSET *stcpu_thread = NULL, *st_duration = NULL;
-            static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL;
-
-            // ----------------------------------------------------------------
-
-            getrusage(RUSAGE_THREAD, &thread);
-
-            if(unlikely(!stcpu_thread)) {
-                stcpu_thread = rrdset_create_localhost(
-                        "netdata"
-                        , "plugin_diskspace"
-                        , NULL
-                        , "diskspace"
-                        , NULL
-                        , "Netdata Disk Space Plugin CPU usage"
-                        , "milliseconds/s"
-                        , PLUGIN_DISKSPACE_NAME
-                        , NULL
-                        , NETDATA_CHART_PRIO_NETDATA_DISKSPACE
-                        , update_every
-                        , RRDSET_TYPE_STACKED
-                );
-
-                rd_user   = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-                rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-            }
-            else
-                rrdset_next(stcpu_thread);
-
-            rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
-            rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
-            rrdset_done(stcpu_thread);
-
-            // ----------------------------------------------------------------
-
-            if(unlikely(!st_duration)) {
-                st_duration = rrdset_create_localhost(
-                        "netdata"
-                        , "plugin_diskspace_dt"
-                        , NULL
-                        , "diskspace"
-                        , NULL
-                        , "Netdata Disk Space Plugin Duration"
-                        , "milliseconds/run"
-                        , PLUGIN_DISKSPACE_NAME
-                        , NULL
-                        , 132021
-                        , update_every
-                        , RRDSET_TYPE_AREA
-                );
-
-                rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
-            }
-            else
-                rrdset_next(st_duration);
-
-            rrddim_set_by_pointer(st_duration, rd_duration, duration);
-            rrdset_done(st_duration);
-
-            // ----------------------------------------------------------------
-
-            if(unlikely(netdata_exit)) break;
         }
+
     }
+    worker_unregister();
 
     netdata_thread_cleanup_pop(1);
     return NULL;
diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c
index eb9332273e..9f411bb8bf 100644
--- a/collectors/ebpf.plugin/ebpf.c
+++ b/collectors/ebpf.plugin/ebpf.c
@@ -1864,6 +1864,8 @@ static void ebpf_manage_pid(pid_t pid)
  */
 int main(int argc, char **argv)
 {
+    clocks_init();
+
     set_global_variables();
     ebpf_parse_args(argc, argv);
     ebpf_manage_pid(getpid());
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
index 97ca1d9ae0..a52ece3f94 100644
--- a/collectors/freebsd.plugin/plugin_freebsd.c
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -9,7 +9,6 @@ static struct freebsd_module {
     int enabled;
 
     int (*func)(int update_every, usec_t dt);
-    usec_t duration;
 
     RRDDIM *rd;
 
@@ -68,8 +67,14 @@ static struct freebsd_module {
     {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
 };
 
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 33
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 33
+#endif
+
 static void freebsd_main_cleanup(void *ptr)
 {
+    worker_unregister();
+
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -80,9 +85,9 @@ static void freebsd_main_cleanup(void *ptr)
 
 void *freebsd_main(void *ptr)
 {
-    netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
+    worker_register("FREEBSD");
 
-    int vdo_cpu_netdata = config_get_boolean("plugin:freebsd", "netdata server resources", 1);
+    netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
 
     // initialize FreeBSD plugin
     if (freebsd_plugin_init())
@@ -94,8 +99,9 @@ void *freebsd_main(void *ptr)
         struct freebsd_module *pm = &freebsd_modules[i];
 
         pm->enabled = config_get_boolean("plugin:freebsd", pm->name, pm->enabled);
-        pm->duration = 0ULL;
         pm->rd = NULL;
+
+        worker_register_job_name(i, freebsd_modules[i].dim);
     }
 
     usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
@@ -103,14 +109,13 @@ void *freebsd_main(void *ptr)
     heartbeat_init(&hb);
 
     while (!netdata_exit) {
+        worker_is_idle();
+
         usec_t hb_dt = heartbeat_next(&hb, step);
-        usec_t duration = 0ULL;
 
         if (unlikely(netdata_exit))
             break;
 
-        // BEGIN -- the job to be done
-
         for (i = 0; freebsd_modules[i].name; i++) {
             struct freebsd_module *pm = &freebsd_modules[i];
             if (unlikely(!pm->enabled))
@@ -118,92 +123,12 @@ void *freebsd_main(void *ptr)
 
             debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
 
+            worker_is_busy(i);
             pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
-            pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
-            duration += pm->duration;
 
             if (unlikely(netdata_exit))
                 break;
         }
-
-        // END -- the job is done
-
-        if (vdo_cpu_netdata) {
-            static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
-            static RRDDIM *rd_user = NULL, *rd_system = NULL;
-
-            // ----------------------------------------------------------------
-
-            struct rusage thread;
-            getrusage(RUSAGE_THREAD, &thread);
-
-            if (unlikely(!st_cpu_thread)) {
-                st_cpu_thread = rrdset_create_localhost(
-                    "netdata",
-                    "plugin_freebsd_cpu",
-                    NULL,
-                    "freebsd",
-                    NULL,
-                    "Netdata FreeBSD plugin CPU usage",
-                    "milliseconds/s",
-                    "freebsd.plugin",
-                    "stats",
-                    132000,
-                    localhost->rrd_update_every,
-                    RRDSET_TYPE_STACKED);
-
-                rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
-                rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
-            } else {
-                rrdset_next(st_cpu_thread);
-            }
-
-            rrddim_set_by_pointer(