Move global stats to a separate thread (#10991)

8 files changed, 408 insertions, 246 deletions

diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index d4cfb1ffe1..ff259d41cf 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -4077,7 +4077,7 @@ void *cgroups_main(void *ptr) {
                         , NULL
                         , "cgroups"
                         , NULL
-                        , "NetData CGroups Plugin CPU usage"
+                        , "Netdata CGroups Plugin CPU usage"
                         , "milliseconds/s"
                         , PLUGIN_CGROUPS_NAME
                         , "stats"
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index e3fc3fde45..311b55adff 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -413,7 +413,7 @@ void *diskspace_main(void *ptr) {
                         , NULL
                         , "diskspace"
                         , NULL
-                        , "NetData Disk Space Plugin CPU usage"
+                        , "Netdata Disk Space Plugin CPU usage"
                         , "milliseconds/s"
                         , PLUGIN_DISKSPACE_NAME
                         , NULL
@@ -441,7 +441,7 @@ void *diskspace_main(void *ptr) {
                         , NULL
                         , "diskspace"
                         , NULL
-                        , "NetData Disk Space Plugin Duration"
+                        , "Netdata Disk Space Plugin Duration"
                         , "milliseconds/run"
                         , PLUGIN_DISKSPACE_NAME
                         , NULL
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
index bee8395f53..17fec41284 100644
--- a/collectors/freebsd.plugin/plugin_freebsd.c
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -15,60 +15,61 @@ static struct freebsd_module {
 
 } freebsd_modules[] = {
 
-        // system metrics
-        { .name = "kern.cp_time",          .dim = "cp_time",        .enabled = 1, .func = do_kern_cp_time },
-        { .name = "vm.loadavg",            .dim = "loadavg",        .enabled = 1, .func = do_vm_loadavg },
-        { .name = "system.ram",            .dim = "system_ram",     .enabled = 1, .func = do_system_ram },
-        { .name = "vm.swap_info",          .dim = "swap",           .enabled = 1, .func = do_vm_swap_info },
-        { .name = "vm.stats.vm.v_swappgs", .dim = "swap_io",        .enabled = 1, .func = do_vm_stats_sys_v_swappgs },
-        { .name = "vm.vmtotal",            .dim = "vmtotal",        .enabled = 1, .func = do_vm_vmtotal },
-        { .name = "vm.stats.vm.v_forks",   .dim = "forks",          .enabled = 1, .func = do_vm_stats_sys_v_forks },
-        { .name = "vm.stats.sys.v_swtch",  .dim = "context_swtch",  .enabled = 1, .func = do_vm_stats_sys_v_swtch },
-        { .name = "hw.intrcnt",            .dim = "hw_intr",        .enabled = 1, .func = do_hw_intcnt },
-        { .name = "vm.stats.sys.v_intr",   .dim = "dev_intr",       .enabled = 1, .func = do_vm_stats_sys_v_intr },
-        { .name = "vm.stats.sys.v_soft",   .dim = "soft_intr",      .enabled = 1, .func = do_vm_stats_sys_v_soft },
-        { .name = "net.isr",               .dim = "net_isr",        .enabled = 1, .func = do_net_isr },
-        { .name = "kern.ipc.sem",          .dim = "semaphores",     .enabled = 1, .func = do_kern_ipc_sem },
-        { .name = "kern.ipc.shm",          .dim = "shared_memory",  .enabled = 1, .func = do_kern_ipc_shm },
-        { .name = "kern.ipc.msq",          .dim = "message_queues", .enabled = 1, .func = do_kern_ipc_msq },
-        { .name = "uptime",                .dim = "uptime",         .enabled = 1, .func = do_uptime },
-
-        // memory metrics
-        { .name = "vm.stats.vm.v_pgfaults", .dim = "pgfaults",      .enabled = 1, .func = do_vm_stats_sys_v_pgfaults },
-
-        // CPU metrics
-        { .name = "kern.cp_times",         .dim = "cp_times",       .enabled = 1, .func = do_kern_cp_times },
-        { .name = "dev.cpu.temperature",   .dim = "cpu_temperature", .enabled = 1, .func = do_dev_cpu_temperature },
-        { .name = "dev.cpu.0.freq",        .dim = "cpu_frequency",  .enabled = 1, .func = do_dev_cpu_0_freq },
-
-        // disk metrics
-        { .name = "kern.devstat",          .dim = "kern_devstat",   .enabled = 1, .func = do_kern_devstat },
-        { .name = "getmntinfo",            .dim = "getmntinfo",     .enabled = 1, .func = do_getmntinfo },
-
-        // network metrics
-        { .name = "net.inet.tcp.states",   .dim = "tcp_states",     .enabled = 1, .func = do_net_inet_tcp_states },
-        { .name = "net.inet.tcp.stats",    .dim = "tcp_stats",      .enabled = 1, .func = do_net_inet_tcp_stats },
-        { .name = "net.inet.udp.stats",    .dim = "udp_stats",      .enabled = 1, .func = do_net_inet_udp_stats },
-        { .name = "net.inet.icmp.stats",   .dim = "icmp_stats",     .enabled = 1, .func = do_net_inet_icmp_stats },
-        { .name = "net.inet.ip.stats",     .dim = "ip_stats",       .enabled = 1, .func = do_net_inet_ip_stats },
-        { .name = "net.inet6.ip6.stats",   .dim = "ip6_stats",      .enabled = 1, .func = do_net_inet6_ip6_stats },
-        { .name = "net.inet6.icmp6.stats", .dim = "icmp6_stats",    .enabled = 1, .func = do_net_inet6_icmp6_stats },
-
-        // network interfaces metrics
-        { .name = "getifaddrs",            .dim = "getifaddrs",   .enabled = 1, .func = do_getifaddrs },
-
-        // ZFS metrics
-        { .name = "kstat.zfs.misc.arcstats", .dim = "arcstats",   .enabled = 1, .func = do_kstat_zfs_misc_arcstats },
-        { .name = "kstat.zfs.misc.zio_trim", .dim = "trim",       .enabled = 1, .func = do_kstat_zfs_misc_zio_trim },
-
-        // ipfw metrics
-        { .name = "ipfw",                  .dim = "ipfw",         .enabled = 1, .func = do_ipfw },
-
-        // the terminator of this array
-        { .name = NULL, .dim = NULL, .enabled = 0, .func = NULL }
+    // system metrics
+    {.name = "kern.cp_time",            .dim = "cp_time",         .enabled = 1, .func = do_kern_cp_time},
+    {.name = "vm.loadavg",              .dim = "loadavg",         .enabled = 1, .func = do_vm_loadavg},
+    {.name = "system.ram",              .dim = "system_ram",      .enabled = 1, .func = do_system_ram},
+    {.name = "vm.swap_info",            .dim = "swap",            .enabled = 1, .func = do_vm_swap_info},
+    {.name = "vm.stats.vm.v_swappgs",   .dim = "swap_io",         .enabled = 1, .func = do_vm_stats_sys_v_swappgs},
+    {.name = "vm.vmtotal",              .dim = "vmtotal",         .enabled = 1, .func = do_vm_vmtotal},
+    {.name = "vm.stats.vm.v_forks",     .dim = "forks",           .enabled = 1, .func = do_vm_stats_sys_v_forks},
+    {.name = "vm.stats.sys.v_swtch",    .dim = "context_swtch",   .enabled = 1, .func = do_vm_stats_sys_v_swtch},
+    {.name = "hw.intrcnt",              .dim = "hw_intr",         .enabled = 1, .func = do_hw_intcnt},
+    {.name = "vm.stats.sys.v_intr",     .dim = "dev_intr",        .enabled = 1, .func = do_vm_stats_sys_v_intr},
+    {.name = "vm.stats.sys.v_soft",     .dim = "soft_intr",       .enabled = 1, .func = do_vm_stats_sys_v_soft},
+    {.name = "net.isr",                 .dim = "net_isr",         .enabled = 1, .func = do_net_isr},
+    {.name = "kern.ipc.sem",            .dim = "semaphores",      .enabled = 1, .func = do_kern_ipc_sem},
+    {.name = "kern.ipc.shm",            .dim = "shared_memory",   .enabled = 1, .func = do_kern_ipc_shm},
+    {.name = "kern.ipc.msq",            .dim = "message_queues",  .enabled = 1, .func = do_kern_ipc_msq},
+    {.name = "uptime",                  .dim = "uptime",          .enabled = 1, .func = do_uptime},
+
+    // memory metrics
+    {.name = "vm.stats.vm.v_pgfaults",  .dim = "pgfaults",        .enabled = 1, .func = do_vm_stats_sys_v_pgfaults},
+
+    // CPU metrics
+    {.name = "kern.cp_times",           .dim = "cp_times",        .enabled = 1, .func = do_kern_cp_times},
+    {.name = "dev.cpu.temperature",     .dim = "cpu_temperature", .enabled = 1, .func = do_dev_cpu_temperature},
+    {.name = "dev.cpu.0.freq",          .dim = "cpu_frequency",   .enabled = 1, .func = do_dev_cpu_0_freq},
+
+    // disk metrics
+    {.name = "kern.devstat",            .dim = "kern_devstat",    .enabled = 1, .func = do_kern_devstat},
+    {.name = "getmntinfo",              .dim = "getmntinfo",      .enabled = 1, .func = do_getmntinfo},
+
+    // network metrics
+    {.name = "net.inet.tcp.states",     .dim = "tcp_states",      .enabled = 1, .func = do_net_inet_tcp_states},
+    {.name = "net.inet.tcp.stats",      .dim = "tcp_stats",       .enabled = 1, .func = do_net_inet_tcp_stats},
+    {.name = "net.inet.udp.stats",      .dim = "udp_stats",       .enabled = 1, .func = do_net_inet_udp_stats},
+    {.name = "net.inet.icmp.stats",     .dim = "icmp_stats",      .enabled = 1, .func = do_net_inet_icmp_stats},
+    {.name = "net.inet.ip.stats",       .dim = "ip_stats",        .enabled = 1, .func = do_net_inet_ip_stats},
+    {.name = "net.inet6.ip6.stats",     .dim = "ip6_stats",       .enabled = 1, .func = do_net_inet6_ip6_stats},
+    {.name = "net.inet6.icmp6.stats",   .dim = "icmp6_stats",     .enabled = 1, .func = do_net_inet6_icmp6_stats},
+
+    // network interfaces metrics
+    {.name = "getifaddrs",              .dim = "getifaddrs",      .enabled = 1, .func = do_getifaddrs},
+
+    // ZFS metrics
+    {.name = "kstat.zfs.misc.arcstats", .dim = "arcstats",        .enabled = 1, .func = do_kstat_zfs_misc_arcstats},
+    {.name = "kstat.zfs.misc.zio_trim", .dim = "trim",            .enabled = 1, .func = do_kstat_zfs_misc_zio_trim},
+
+    // ipfw metrics
+    {.name = "ipfw",                    .dim = "ipfw",            .enabled = 1, .func = do_ipfw},
+
+    // the terminator of this array
+    {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
 };
 
-static void freebsd_main_cleanup(void *ptr) {
+static void freebsd_main_cleanup(void *ptr)
+{
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -77,7 +78,8 @@ static void freebsd_main_cleanup(void *ptr) {
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
-void *freebsd_main(void *ptr) {
+void *freebsd_main(void *ptr)
+{
     netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
 
     int vdo_cpu_netdata = config_get_boolean("plugin:freebsd", "netdata server resources", 1);
@@ -88,7 +90,7 @@ void *freebsd_main(void *ptr) {
 
     // check the enabled status for each module
     int i;
-    for(i = 0 ; freebsd_modules[i].name ;i++) {
+    for (i = 0; freebsd_modules[i].name; i++) {
         struct freebsd_module *pm = &freebsd_modules[i];
 
         pm->enabled = config_get_boolean("plugin:freebsd", pm->name, pm->enabled);
@@ -100,17 +102,19 @@ void *freebsd_main(void *ptr) {
     heartbeat_t hb;
     heartbeat_init(&hb);
 
-    while(!netdata_exit) {
+    while (!netdata_exit) {
         usec_t hb_dt = heartbeat_next(&hb, step);
         usec_t duration = 0ULL;
 
-        if(unlikely(netdata_exit)) break;
+        if (unlikely(netdata_exit))
+            break;
 
         // BEGIN -- the job to be done
 
-        for(i = 0 ; freebsd_modules[i].name ;i++) {
+        for (i = 0; freebsd_modules[i].name; i++) {
             struct freebsd_module *pm = &freebsd_modules[i];
-            if(unlikely(!pm->enabled)) continue;
+            if (unlikely(!pm->enabled))
+                continue;
 
             debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
 
@@ -118,55 +122,87 @@ void *freebsd_main(void *ptr) {
             pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
             duration += pm->duration;
 
-            if(unlikely(netdata_exit)) break;
+            if (unlikely(netdata_exit))
+                break;
         }
 
         // END -- the job is done
 
-        // --------------------------------------------------------------------
-
-        if(vdo_cpu_netdata) {
-            static RRDSET *st = NULL;
-
-            if(unlikely(!st)) {
-                st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
-
-                if(!st) {
-                    st = rrdset_create_localhost(
-                            "netdata"
-                            , "plugin_freebsd_modules"
-                            , NULL
-                            , "freebsd"
-                            , NULL
-                            , "NetData FreeBSD Plugin Modules Durations"
-                            , "milliseconds/run"
-                            , "netdata"
-                            , "stats"
-                            , 132001
-                            , localhost->rrd_update_every
-                            , RRDSET_TYPE_STACKED
-                    );
-
-                    for(i = 0 ; freebsd_modules[i].name ;i++) {
+        if (vdo_cpu_netdata) {
+            static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
+            static RRDDIM *rd_user = NULL, *rd_system = NULL;
+
+            // ----------------------------------------------------------------
+
+            struct rusage thread;
+            getrusage(RUSAGE_THREAD, &thread);
+
+            if (unlikely(!st_cpu_thread)) {
+                st_cpu_thread = rrdset_create_localhost(
+                    "netdata",
+                    "plugin_freebsd_cpu",
+                    NULL,
+                    "freebsd",
+                    NULL,
+                    "Netdata FreeBSD plugin CPU usage",
+                    "milliseconds/s",
+                    "freebsd",
+                    "stats",
+                    132000,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_STACKED);
+
+                rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+                rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+            } else {
+                rrdset_next(st_cpu_thread);
+            }
+
+            rrddim_set_by_pointer(
+                st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
+            rrddim_set_by_pointer(
+                st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
+            rrdset_done(st_cpu_thread);
+
+            // ----------------------------------------------------------------
+
+            if (unlikely(!st_duration)) {
+                st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
+
+                if (!st_duration) {
+                    st_duration = rrdset_create_localhost(
+                        "netdata",
+                        "plugin_freebsd_modules",
+                        NULL,
+                        "freebsd",
+                        NULL,
+                        "Netdata FreeBSD plugin modules durations",
+                        "milliseconds/run",
+                        "freebsd",
+                        "stats",
+                        132001,
+                        localhost->rrd_update_every,
+                        RRDSET_TYPE_STACKED);
+
+                    for (i = 0; freebsd_modules[i].name; i++) {
                         struct freebsd_module *pm = &freebsd_modules[i];
-                        if(unlikely(!pm->enabled)) continue;
+                        if (unlikely(!pm->enabled))
+                            continue;
 
-                        pm->rd = rrddim_add(st, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+                        pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
                     }
                 }
-            }
-            else rrdset_next(st);
+            } else
+                rrdset_next(st_duration);
 
-            for(i = 0 ; freebsd_modules[i].name ;i++) {
+            for (i = 0; freebsd_modules[i].name; i++) {
                 struct freebsd_module *pm = &freebsd_modules[i];
-                if(unlikely(!pm->enabled)) continue;
+                if (unlikely(!pm->enabled))
+                    continue;
 
-                rrddim_set_by_pointer(st, pm->rd, pm->duration);
+                rrddim_set_by_pointer(st_duration, pm->rd, pm->duration);
             }
-            rrdset_done(st);
-
-            global_statistics_charts();
-            registry_statistics();
+            rrdset_done(st_duration);
         }
     }
 
diff --git a/collectors/macos.plugin/plugin_macos.c b/collectors/macos.plugin/plugin_macos.c
index 628a5b10dc..1a64ed81c9 100644
--- a/collectors/macos.plugin/plugin_macos.c
+++ b/collectors/macos.plugin/plugin_macos.c
@@ -2,7 +2,28 @@
 
 #include "plugin_macos.h"
 
-static void macos_main_cleanup(void *ptr) {
+static struct macos_module {
+    const char *name;
+    const char *dim;
+
+    int enabled;
+
+    int (*func)(int update_every, usec_t dt);
+    usec_t duration;
+
+    RRDDIM *rd;
+
+} macos_modules[] = {
+    {.name = "sysctl",                           .dim = "sysctl",   .enabled = 1, .func = do_macos_sysctl},
+    {.name = "mach system management interface", .dim = "mach_smi", .enabled = 1, .func = do_macos_mach_smi},
+    {.name = "iokit",                            .dim = "iokit",    .enabled = 1, .func = do_macos_iokit},
+
+    // the terminator of this array
+    {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
+};
+
+static void macos_main_cleanup(void *ptr)
+{
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -11,56 +32,123 @@ static void macos_main_cleanup(void *ptr) {
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
-void *macos_main(void *ptr) {
+void *macos_main(void *ptr)
+{
     netdata_thread_cleanup_push(macos_main_cleanup, ptr);
 
-    // when ZERO, attempt to do it
-    int vdo_cpu_netdata             = !config_get_boolean("plugin:macos", "netdata server resources", 1);
-    int vdo_macos_sysctl            = !config_get_boolean("plugin:macos", "sysctl", 1);
-    int vdo_macos_mach_smi          = !config_get_boolean("plugin:macos", "mach system management interface", 1);
-    int vdo_macos_iokit             = !config_get_boolean("plugin:macos", "iokit", 1);
+    int vdo_cpu_netdata = config_get_boolean("plugin:macos", "netdata server resources", CONFIG_BOOLEAN_YES);
 
-    // keep track of the time each module was called
-    unsigned long long sutime_macos_sysctl = 0ULL;
-    unsigned long long sutime_macos_mach_smi = 0ULL;
-    unsigned long long sutime_macos_iokit = 0ULL;
+    // check the enabled status for each module
+    for (int i = 0; macos_modules[i].name; i++) {
+        struct macos_module *pm = &macos_modules[i];
+
+        pm->enabled = config_get_boolean("plugin:macos", pm->name, pm->enabled);
+        pm->duration = 0ULL;
+        pm->rd = NULL;
+    }
 
     usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
     heartbeat_t hb;
     heartbeat_init(&hb);
 
-    while(!netdata_exit) {
+    while (!netdata_exit) {
         usec_t hb_dt = heartbeat_next(&hb, step);
-
-        if(unlikely(netdata_exit)) break;
+        usec_t duration = 0ULL;
 
         // BEGIN -- the job to be done
 
-        if(!vdo_macos_sysctl) {
-            debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_sysctl().");
-            vdo_macos_sysctl = do_macos_sysctl(localhost->rrd_update_every, hb_dt);
-        }
-        if(unlikely(netdata_exit)) break;
+        for (int i = 0; macos_modules[i].name; i++) {
+            struct macos_module *pm = &macos_modules[i];
+            if (unlikely(!pm->enabled))
+                continue;
 
-        if(!vdo_macos_mach_smi) {
-            debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_mach_smi().");
-            vdo_macos_mach_smi = do_macos_mach_smi(localhost->rrd_update_every, hb_dt);
-        }
-        if(unlikely(netdata_exit)) break;
+            debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name);
+
+            pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
+            pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
+            duration += pm->duration;
 
-        if(!vdo_macos_iokit) {
-            debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_iokit().");
-            vdo_macos_iokit = do_macos_iokit(localhost->rrd_update_every, hb_dt);
+            if (unlikely(netdata_exit))
+                break;
         }
-        if(unlikely(netdata_exit)) break;
 
         // END -- the job is done
 
-        // --------------------------------------------------------------------
-
-        if(!vdo_cpu_netdata) {
-            global_statistics_charts();
-            registry_statistics();
+        if (vdo_cpu_netdata) {
+            static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
+            static RRDDIM *rd_user = NULL, *rd_system = NULL;
+
+            // ----------------------------------------------------------------
+
+            struct rusage thread;
+            getrusage(RUSAGE_THREAD, &thread);
+
+            if (unlikely(!st_cpu_thread)) {
+                st_cpu_thread = rrdset_create_localhost(
+                    "netdata",
+                    "plugin_macos_cpu",
+                    NULL,
+                    "macos",
+                    NULL,
+                    "Netdata macOS plugin CPU usage",
+                    "milliseconds/s",
+                    "macos",
+                    "stats",
+                    132000,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_STACKED);
+
+                rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+                rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+            } else {
+                rrdset_next(st_cpu_thread);
+            }
+
+            rrddim_set_by_pointer(
+                st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
+            rrddim_set_by_pointer(
+                st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
+            rrdset_done(st_cpu_thread);
+
+            // ----------------------------------------------------------------
+
+            if (unlikely(!st_duration)) {
+                st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_macos_modules");
+
+                if (!st_duration) {
+                    st_duration = rrdset_create_localhost(
+                        "netdata",
+                        "plugin_macos_modules",
+                        NULL,
+                        "macos",
+                        NULL,
+                        "Netdata macOS plugin modules durations",
+                        "milliseconds/run",
+                        "macos",
+                        "stats",
+                        132001,
+                        localhost->rrd_update_every,
+                        RRDSET_TYPE_STACKED);
+
+                    for (int i = 0; macos_modules[i].name; i++) {
+                        struct macos_module *pm = &macos_modules[i];
+                        if (unlikely(!pm->enabled))
+                            continue;
+
+                        pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+                    }
+                }
+            } else
+                rrdset_next(st_duration);
+
+            for (int i = 0; macos_modules[i].name; i++) {
+                struct macos_module *pm = &macos_modules[i];
+                if (unlikely(!pm->enabled))
+                    continue;
+
+                rrddim_set_by_pointer(st_duration, pm->rd, pm->duration);
+            }
+            rrdset_done(st_duration);
         }
     }
 
diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c
index 19230c09d5..254722069a 100644
--- a/collectors/proc.plugin/plugin_proc.c
+++ b/collectors/proc.plugin/plugin_proc.c
@@ -15,70 +15,73 @@ static struct proc_module {
 
 } proc_modules[] = {
 
-        // system metrics
-        { .name = "/proc/stat", .dim = "stat", .func = do_proc_stat },
-        { .name = "/proc/uptime", .dim = "uptime", .func = do_proc_uptime },
-        { .name = "/proc/loadavg", .dim = "loadavg", .func = do_proc_loadavg },
-        { .name = "/proc/sys/kernel/random/entropy_avail", .dim = "entropy", .func = do_proc_sys_kernel_random_entropy_avail },
-
-        // pressure metrics
-        { .name = "/proc/pressure", .dim = "pressure", .func = do_proc_pressure },
-
-        // CPU metrics
-        { .name = "/proc/interrupts", .dim = "interrupts", .func = do_proc_interrupts },
-        { .name = "/proc/softirqs", .dim = "softirqs", .func = do_proc_softirqs },
-
-        // memory metrics
-        { .name = "/proc/vmstat", .dim = "vmstat", .func = do_proc_vmstat },
-        { .name = "/proc/meminfo", .dim = "meminfo", .func = do_proc_meminfo },
-        { .name = "/sys/kernel/mm/ksm", .dim = "ksm", .func = do_sys_kernel_mm_ksm },
-        { .name = "/sys/block/zram", .dim = "zram", .func = do_sys_block_zram },
-        { .name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc },
-        { .name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node },
-        { .name = "/proc/pagetypeinfo", .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo },
-
-        // network metrics
-        { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev },
-        { .name = "/proc/net/wireless", .dim = "netwireless", .func = do_proc_net_wireless },
-        { .name = "/proc/net/sockstat", .dim = "sockstat", .func = do_proc_net_sockstat },
-        { .name = "/proc/net/sockstat6", .dim = "sockstat6", .func = do_proc_net_sockstat6 },
-        { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, // this has to be before /proc/net/snmp, because there is a shared metric
-        { .name = "/proc/net/snmp", .dim = "snmp", .func = do_proc_net_snmp },
-        { .name = "/proc/net/snmp6", .dim = "snmp6", .func = do_proc_net_snmp6 },
-        { .name = "/proc/net/sctp/snmp", .dim = "sctp", .func = do_proc_net_sctp_snmp },
-        { .name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat },
-        { .name = "/proc/net/ip_vs/stats", .dim = "ipvs", .func = do_proc_net_ip_vs_stats },
-        { .name = "/sys/class/infiniband",   .dim = "infiniband", .func = do_sys_class_infiniband },
-
-        // firewall metrics
-        { .name = "/proc/net/stat/conntrack", .dim = "conntrack", .func = do_proc_net_stat_conntrack },
-        { .name = "/proc/net/stat/synproxy", .dim = "synproxy", .func = do_proc_net_stat_synproxy },
-
-        // disk metrics
-        { .name = "/proc/diskstats", .dim = "diskstats", .func = do_proc_diskstats },
-        { .name = "/proc/mdstat", .dim = "mdstat", .func = do_proc_mdstat },
-
-        // NFS metrics
-        { .name = "/proc/net/rpc/nfsd", .dim = "nfsd", .func = do_proc_net_rpc_nfsd },
-        { .name = "/proc/net/rpc/nfs", .dim = "nfs", .func = do_proc_net_rpc_nfs },
-
-        // ZFS metrics
-        { .name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats },
-
-        // BTRFS metrics
-        { .name = "/sys/fs/btrfs", .dim = "btrfs", .func = do_sys_fs_btrfs },
-
-        // IPC metrics
-        { .name = "ipc", .dim = "ipc", .func = do_ipc },
-
-        // linux power supply metrics
-        { .name = "/sys/class/power_supply", .dim = "power_supply", .func = do_sys_class_power_supply },
-
-        // the terminator of this array
-        { .name = NULL, .dim = NULL, .func = NULL }
+    // system metrics
+    {.name = "/proc/stat",                   .dim = "stat",         .func = do_proc_stat},
+    {.name = "/proc/uptime",                 .dim = "uptime",       .func = do_proc_uptime},
+    {.name = "/proc/loadavg",                .dim = "loadavg",      .func = do_proc_loadavg},
+    {.name = "/proc/sys/kernel/random/entropy_avail", .dim = "entropy", .func = do_proc_sys_kernel_random_entropy_avail},
+
+    // pressure metrics
+    {.name = "/proc/pressure",               .dim = "pressure",     .func = do_proc_pressure},
+
+    // CPU metrics
+    {.name = "/proc/interrupts",             .dim = "interrupts",   .func = do_proc_interrupts},
+    {.name = "/proc/softirqs",               .dim = "softirqs",     .func = do_proc_softirqs},
+
+    // memory metrics
+    {.name = "/proc/vmstat",                 .dim = "vmstat",       .func = do_proc_vmstat},
+    {.name = "/proc/meminfo",                .dim = "meminfo",      .func = do_proc_meminfo},
+    {.name = "/sys/kernel/mm/ksm",           .dim = "ksm",          .func = do_sys_kernel_mm_ksm},
+    {.name = "/sys/block/zram",              .dim = "zram",         .func = do_sys_block_zram},
+    {.name = "/sys/devices/system/edac/mc",  .dim = "ecc",          .func = do_proc_sys_devices_system_edac_mc},
+    {.name = "/sys/devices/system/node",     .dim = "numa",         .func = do_proc_sys_devices_system_node},
+    {.name = "/proc/pagetypeinfo",           .dim = "pagetypeinfo", .func = do_proc_pagetypeinfo},
+
+    // network metrics
+    {.name = "/proc/net/dev",                .dim = "netdev",       .func = do_proc_net_dev},
+    {.name = "/proc/net/wireless",           .dim = "netwireless",  .func = do_proc_net_wireless},
+    {.name = "/proc/net/sockstat",           .dim = "sockstat",     .func = do_proc_net_sockstat},
+    {.name = "/proc/net/sockstat6",          .dim = "sockstat6",    .func = do_proc_net_sockstat6},
+    {.name = "/proc/net/netstat",
+     .dim = "netstat",
+     .func = do_proc_net_netstat}, // this has to be before /proc/net/snmp, because there is a shared metric
+    {.name = "/proc/net/snmp",               .dim = "snmp",         .func = do_proc_net_snmp},
+    {.name = "/proc/net/snmp6",              .dim = "snmp6",        .func = do_proc_net_snmp6},
+    {.name = "/proc/net/sctp/snmp",          .dim = "sctp",         .func = do_proc_net_sctp_snmp},
+    {.name = "/proc/net/softnet_stat",       .dim = "softnet",      .func = do_proc_net_softnet_stat},
+    {.name = "/proc/net/ip_vs/stats",        .dim = "ipvs",         .func = do_proc_net_ip_vs_stats},
+    {.name = "/sys/class/infiniband",        .dim = "infiniband",   .func = do_sys_class_infiniband},
+
+    // firewall metrics
+    {.name = "/proc/net/stat/conntrack",     .dim = "conntrack",    .func = do_proc_net_stat_conntrack},
+    {.name = "/proc/net/stat/synproxy",      .dim = "synproxy",     .func = do_proc_net_stat_synproxy},
+
+    // disk metrics
+    {.name = "/proc/diskstats",              .dim = "diskstats",    .func = do_proc_diskstats},
+    {.name = "/proc/mdstat",                 .dim = "mdstat",       .func = do_proc_mdstat},
+
+    // NFS metrics
+    {.name = "/proc/net/rpc/nfsd",           .dim = "nfsd",         .func = do_proc_net_rpc_nfsd},
+    {.name = "/proc/net/rpc/nfs",            .dim = "nfs",          .func = do_proc_net_rpc_nfs},
+
+    // ZFS metrics
+    {.name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats},
+
+    // BTRFS metrics
+    {.name = "/sys/fs/btrfs",                .dim = "btrfs",        .func = do_sys_fs_btrfs},
+
+    // IPC metrics
+    {.name = "ipc",                          .dim = "ipc",          .func = do_ipc},
+
+    {.name = "/sys/class/power_supply",      .dim = "power_supply", .func = do_sys_class_power_supply},
+    // linux power supply metrics
+
+    // the terminator of this array
+    {.name = NULL, .dim = NULL, .func = NULL}
 };
 
-static void proc_main_cleanup(void *ptr) {
+static void proc_main_cleanup(void *ptr)
+{
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
 
@@ -87,7 +90,8 @@ static void proc_main_cleanup(void *ptr) {
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
-void *proc_main(void *ptr) {
+void *proc_main(void *ptr)
+{
     netdata_thread_cleanup_push(proc_main_cleanup, ptr);
 
     int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", CONFIG_BOOLEAN_YES);
@@ -96,7 +100,7 @@ void *proc_main(void *ptr) {
 
     // check the enabled status for each module
     int i;
-    for(i = 0 ; proc_modules[i].name ;i++) {
+    for (i = 0; proc_modules[i].name; i++) {
         struct proc_module *pm = &proc_modules[i];
 
         pm->enabled = config_get_boolean("plugin:proc", pm->name, CONFIG_BOOLEAN_YES);
@@ -109,20 +113,22 @@ void *proc_main(void *ptr) {
     heartbeat_init(&hb);
     size_t iterations = 0;
 
-    while(!netdata_exit) {
+    while (!netdata_exit) {
         iterations++;
         (void)iterations;
 
         usec_t hb_dt = heartbeat_next(&hb, step);
         usec_t duration = 0ULL;
 
-        if(unlikely(netdata_exit)) break;
+        if (unlikely(netdata_exit))
+            break;
 
         // BEGIN -- the job to be done
 
-        for(i = 0 ; proc_modules[i].name ;i++) {
+        for (i = 0; proc_modules[i].name; i++) {
             struct proc_module *pm = &proc_modules[i];
-            if(unlikely(!pm->enabled)) continue;
+            if (unlikely(!pm->enabled))
+                continue;
 
             debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name);
 
@@ -139,55 +145,87 @@ void *proc_main(void *ptr) {
 //                log_thread_memory_allocations = 0;
 //#endif
 
-            if(unlikely(netdata_exit)) break;
+            if (unlikely(netdata_exit))
+                break;
         }
 
         // END -- the job is done
 
-        // --------------------------------------------------------------------
-
-        if(vdo_cpu_netdata) {
-            static RRDSET *st = NULL;
-
-            if(unlikely(!st)) {
-                st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
-
-                if(!st) {
-                    st = rrdset_create_localhost(
-                            "netdata"
-                            , "plugin_proc_modules"
-                            , NULL
-                            , "proc"
-                            , NULL
-                            , "NetData Proc Plugin Modules Durations"
-                            , "milliseconds/run"
-                            , "netdata"
-                            , "stats"
-                            , 132001
-                            , localhost->rrd_update_every
-                            , RRDSET_TYPE_STACKED
-                    );
-
-                    for(i = 0 ; proc_modules[i].name ;i++) {
+        if (vdo_cpu_netdata) {
+            static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
+            static RRDDIM *rd_user = NULL, *rd_system = NULL;
+
+            // ----------------------------------------------------------------
+
+            struct rusage thread;
+            getrusage(RUSAGE_THREAD, &thread);
+
+            if (unlikely(!st_cpu_thread)) {
+                st_cpu_thread = rrdset_create_localhost(
+