Split cgroup charts (#15975)

Co-authored-by: ilyam8 <ilya@netdata.cloud>

2 files changed, 507 insertions, 884 deletions

diff --git a/collectors/cgroups.plugin/metadata.yaml b/collectors/cgroups.plugin/metadata.yaml
index b342d30a3b..64446c226b 100644
--- a/collectors/cgroups.plugin/metadata.yaml
+++ b/collectors/cgroups.plugin/metadata.yaml
@@ -821,154 +821,104 @@ modules:
       description: ""
       availability: []
       scopes:
-        - name: global
+        - name: systemd service
           description: ""
-          labels: []
+          labels:
+            - name: service_name
+              description: Service name
           metrics:
-            - name: services.cpu
+            - name: systemd.service.cpu.utilization
               description: Systemd Services CPU utilization (100% = 1 core)
-              unit: "percentage"
+              unit: percentage
               chart_type: stacked
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_usage
+                - name: user
+                - name: system
+            - name: systemd.service.memory.usage
               description: Systemd Services Used Memory
-              unit: "MiB"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_rss
-              description: Systemd Services RSS Memory
-              unit: "MiB"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_mapped
-              description: Systemd Services Mapped Memory
-              unit: "MiB"
+              unit: MiB
               chart_type: stacked
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_cache
-              description: Systemd Services Cache Memory
-              unit: "MiB"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_writeback
-              description: Systemd Services Writeback Memory
-              unit: "MiB"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_pgfault
-              description: Systemd Services Memory Minor Page Faults
-              unit: "MiB/s"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_pgmajfault
-              description: Systemd Services Memory Major Page Faults
-              unit: "MiB/s"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_pgpgin
-              description: Systemd Services Memory Charging Activity
-              unit: "MiB/s"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_pgpgout
-              description: Systemd Services Memory Uncharging Activity
-              unit: "MiB/s"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.mem_failcnt
+                - name: ram
+                - name: swap
+            - name: systemd.service.memory.failcnt
               description: Systemd Services Memory Limit Failures
-              unit: "failures"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.swap_usage
-              description: Systemd Services Swap Memory Used
-              unit: "MiB"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.io_read
-              description: Systemd Services Disk Read Bandwidth
-              unit: "KiB/s"
-              chart_type: stacked
-              dimensions:
-                - name: a dimension per systemd service
-            - name: services.io_write
-              description: Systemd Services Disk Write Bandwidth
-              unit: "KiB/s"
-              chart_type: stacked
+              unit: failures/s
+              chart_type: line
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.io_ops_read
-              description: Systemd Services Disk Read Operations
-              unit: "operations/s"
+                - name: fail
+            - name: systemd.service.memory.ram.usage
+              description: Systemd Services Memory
+              unit: MiB
               chart_type: stacked
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.io_ops_write
-              description: Systemd Services Disk Write Operations
-              unit: "operations/s"
+                - name: rss
+                - name: cache
+                - name: mapped_file
+                - name: rss_huge
+            - name: systemd.service.memory.writeback
+              description: Systemd Services Writeback Memory
+              unit: MiB
               chart_type: stacked
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.throttle_io_read
-              description: Systemd Services Throttle Disk Read Bandwidth
-              unit: "KiB/s"
-              chart_type: stacked
+                - name: writeback
+                - name: dirty
+            - name: systemd.service.memory.paging.faults
+              description: Systemd Services Memory Minor and Major Page Faults
+              unit: MiB/s
+              chart_type: area
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.services.throttle_io_write
-              description: Systemd Services Throttle Disk Write Bandwidth
-              unit: "KiB/s"
-              chart_type: stacked
+                - name: minor
+                - name: major
+            - name: systemd.service.memory.paging.io
+              description: Systemd Services Memory Paging IO
+              unit: MiB/s
+              chart_type: area
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.throttle_io_ops_read
-              description: Systemd Services Throttle Disk Read Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: in
+                - name: out
+            - name: systemd.service.disk.io
+              description: Systemd Services Disk Read/Write Bandwidth
+              unit: KiB/s
+              chart_type: area
               dimensions:
-                - name: a dimension per systemd service
-            - name: throttle_io_ops_write
-              description: Systemd Services Throttle Disk Write Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: read
+                - name: write
+            - name: systemd.service.disk.iops
+              description: Systemd Services Disk Read/Write Operations
+              unit: operations/s
+              chart_type: line
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.queued_io_ops_read
-              description: Systemd Services Queued Disk Read Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: read
+                - name: write
+            - name: systemd.service.disk.throttle.io
+              description: Systemd Services Throttle Disk Read/Write Bandwidth
+              unit: KiB/s
+              chart_type: area
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.queued_io_ops_write
-              description: Systemd Services Queued Disk Write Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: read
+                - name: write
+            - name: systemd.service.disk.throttle.iops
+              description: Systemd Services Throttle Disk Read/Write Operations
+              unit: operations/s
+              chart_type: line
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.merged_io_ops_read
-              description: Systemd Services Merged Disk Read Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: read
+                - name: write
+            - name: systemd.service.disk.queued_iops
+              description: Systemd Services Queued Disk Read/Write Operations
+              unit: operations/s
+              chart_type: line
               dimensions:
-                - name: a dimension per systemd service
-            - name: services.merged_io_ops_write
-              description: Systemd Services Merged Disk Write Operations
-              unit: "operations/s"
-              chart_type: stacked
+                - name: read
+                - name: write
+            - name: systemd.service.disk.merged_iops
+              description: Systemd Services Merged Disk Read/Write Operations
+              unit: operations/s
+              chart_type: line
               dimensions:
-                - name: a dimension per systemd service
+                - name: read
+                - name: write
   - <<: *module
     meta:
       <<: *meta
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index 893a9445d9..d8dbc6e0a5 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -38,6 +38,7 @@
 // cgroup globals
 
 static char cgroup_chart_id_prefix[] = "cgroup_";
+static char services_chart_id_prefix[] = "systemd_";
 
 static int is_inside_k8s = 0;
 
@@ -796,15 +797,18 @@ struct cgroup {
     char enabled;        // enabled in the config
 
     char pending_renames;
-    char *intermediate_id; // TODO: remove it when the renaming script is fixed
 
     char *id;
     uint32_t hash;
 
+    char *intermediate_id; // TODO: remove it when the renaming script is fixed
+
     char *chart_id;
-    uint32_t hash_chart;
+    uint32_t hash_chart_id;
 
-    char *chart_title;
+    // 'cgroup_name' label value.
+    // By default this is the *id (path), later changed to the resolved name (cgroup-name.sh) or systemd service name.
+    char *name;
 
     RRDLABELS *chart_labels;
 
@@ -878,35 +882,6 @@ struct cgroup {
     unsigned long long memoryswap_limit;
     const RRDSETVAR_ACQUIRED *chart_var_memoryswap_limit;
 
-    // services
-    RRDDIM *rd_cpu;
-    RRDDIM *rd_mem_usage;
-    RRDDIM *rd_mem_failcnt;
-    RRDDIM *rd_swap_usage;
-
-    RRDDIM *rd_mem_detailed_cache;
-    RRDDIM *rd_mem_detailed_rss;
-    RRDDIM *rd_mem_detailed_mapped;
-    RRDDIM *rd_mem_detailed_writeback;
-    RRDDIM *rd_mem_detailed_pgpgin;
-    RRDDIM *rd_mem_detailed_pgpgout;
-    RRDDIM *rd_mem_detailed_pgfault;
-    RRDDIM *rd_mem_detailed_pgmajfault;
-
-    RRDDIM *rd_io_service_bytes_read;
-    RRDDIM *rd_io_serviced_read;
-    RRDDIM *rd_throttle_io_read;
-    RRDDIM *rd_throttle_io_serviced_read;
-    RRDDIM *rd_io_queued_read;
-    RRDDIM *rd_io_merged_read;
-
-    RRDDIM *rd_io_service_bytes_write;
-    RRDDIM *rd_io_serviced_write;
-    RRDDIM *rd_throttle_io_write;
-    RRDDIM *rd_throttle_io_serviced_write;
-    RRDDIM *rd_io_queued_write;
-    RRDDIM *rd_io_merged_write;
-
     struct cgroup *next;
     struct cgroup *discovered_next;
 
@@ -1667,7 +1642,7 @@ static inline void read_all_discovered_cgroups(struct cgroup *root) {
 
 #define CGROUP_NETWORK_INTERFACE_MAX_LINE 2048
 static inline void read_cgroup_network_interfaces(struct cgroup *cg) {
-    netdata_log_debug(D_CGROUP, "looking for the network interfaces of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title);
+    netdata_log_debug(D_CGROUP, "looking for the network interfaces of cgroup '%s' with chart id '%s'", cg->id, cg->chart_id);
 
     pid_t cgroup_pid;
     char cgroup_identifier[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1];
@@ -1747,17 +1722,6 @@ static inline void free_cgroup_network_interfaces(struct cgroup *cg) {
 
 #define CGROUP_CHARTID_LINE_MAX 1024
 
-static inline char *cgroup_title_strdupz(const char *s) {
-    if(!s || !*s) s = "/";
-
-    if(*s == '/' && s[1] != '\0') s++;
-
-    char *r = strdupz(s);
-    netdata_fix_chart_name(r);
-
-    return r;
-}
-
 static inline char *cgroup_chart_id_strdupz(const char *s) {
     if(!s || !*s) s = "/";
 
@@ -1866,7 +1830,7 @@ static inline void cgroup_free(struct cgroup *cg) {
     freez(cg->id);
     freez(cg->intermediate_id);
     freez(cg->chart_id);
-    freez(cg->chart_title);
+    freez(cg->name);
 
     rrdlabels_destroy(cg->chart_labels);
 
@@ -1883,7 +1847,7 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {
     }
     cg->pending_renames--;
 
-    netdata_log_debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title);
+    netdata_log_debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s'", cg->id, cg->chart_id);
     netdata_log_debug(D_CGROUP, "executing command %s \"%s\" for cgroup '%s'", cgroups_rename_script, cg->intermediate_id, cg->chart_id);
     pid_t cgroup_pid;
 
@@ -1927,14 +1891,14 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {
     name = cgroup_parse_resolved_name_and_labels(cg->chart_labels, new_name);
     rrdlabels_remove_all_unmarked(cg->chart_labels);
 
-    freez(cg->chart_title);
-    cg->chart_title = cgroup_title_strdupz(name);
+    freez(cg->name);
+    cg->name = strdupz(name);
 
     freez(cg->chart_id);
     cg->chart_id = cgroup_chart_id_strdupz(name);
 
     substitute_dots_in_id(cg->chart_id);
-    cg->hash_chart = simple_hash(cg->chart_id);
+    cg->hash_chart_id = simple_hash(cg->chart_id);
 }
 
 static void is_cgroup_procs_exist(netdata_ebpf_cgroup_shm_body_t *out, char *id) {
@@ -1992,21 +1956,31 @@ static inline void convert_cgroup_to_systemd_service(struct cgroup *cg) {
         s[len] = '\0';
     }
 
-    freez(cg->chart_title);
-    cg->chart_title = cgroup_title_strdupz(s);
+    freez(cg->name);
+    cg->name = strdupz(s);
+
+    freez(cg->chart_id);
+    cg->chart_id = cgroup_chart_id_strdupz(s);
+    substitute_dots_in_id(cg->chart_id);
+    cg->hash_chart_id = simple_hash(cg->chart_id);
 }
 
 static inline struct cgroup *discovery_cgroup_add(const char *id) {
     netdata_log_debug(D_CGROUP, "adding to list, cgroup with id '%s'", id);
 
     struct cgroup *cg = callocz(1, sizeof(struct cgroup));
+
     cg->id = strdupz(id);
     cg->hash = simple_hash(cg->id);
-    cg->chart_title = cgroup_title_strdupz(id);
+
+    cg->name = strdupz(id);
+
     cg->intermediate_id = cgroup_chart_id_strdupz(id);
+
     cg->chart_id = cgroup_chart_id_strdupz(id);
     substitute_dots_in_id(cg->chart_id);
-    cg->hash_chart = simple_hash(cg->chart_id);
+    cg->hash_chart_id = simple_hash(cg->chart_id);
+
     if (cgroup_use_unified_cgroups) {
         cg->options |= CGROUP_OPTIONS_IS_UNIFIED;
     }
@@ -2500,8 +2474,10 @@ static inline void discovery_cleanup_all_cgroups() {
             // enable the first duplicate cgroup
             {
                 struct cgroup *t;
-                for(t = discovered_cgroup_root; t ; t = t->discovered_next) {
-                    if(t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE && t->hash_chart == cg->hash_chart && !strcmp(t->chart_id, cg->chart_id)) {
+                for (t = discovered_cgroup_root; t; t = t->discovered_next) {
+                    if (t != cg && t->available && !t->enabled && t->options & CGROUP_OPTIONS_DISABLED_DUPLICATE &&
+                        (is_cgroup_systemd_service(t) == is_cgroup_systemd_service(cg)) &&
+                        t->hash_chart_id == cg->hash_chart_id && !strcmp(t->chart_id, cg->chart_id)) {
                         netdata_log_debug(D_CGROUP, "Enabling duplicate of cgroup '%s' with id '%s', because the original with id '%s' stopped.", t->chart_id, t->id, cg->id);
                         t->enabled = 1;
                         t->options &= ~CGROUP_OPTIONS_DISABLED_DUPLICATE;
@@ -2553,8 +2529,8 @@ static inline void discovery_share_cgroups_with_ebpf() {
 
     for (cg = cgroup_root, count = 0; cg; cg = cg->next, count++) {
         netdata_ebpf_cgroup_shm_body_t *ptr = &shm_cgroup_ebpf.body[count];
-        char *prefix = (is_cgroup_systemd_service(cg)) ? "" : "cgroup_";
-        snprintfz(ptr->name, CGROUP_EBPF_NAME_SHARED_LENGTH - 1, "%s%s", prefix, cg->chart_title);
+        char *prefix = (is_cgroup_systemd_service(cg)) ? services_chart_id_prefix : cgroup_chart_id_prefix;
+        snprintfz(ptr->name, CGROUP_EBPF_NAME_SHARED_LENGTH - 1, "%s%s", prefix, cg->chart_id);
         ptr->hash = simple_hash(ptr->name);
         ptr->options = cg->options;
         ptr->enabled = cg->enabled;
@@ -2658,13 +2634,13 @@ static inline void discovery_process_first_time_seen_cgroup(struct cgroup *cg) {
     }
 
     if (cgroup_enable_systemd_services && matches_systemd_services_cgroups(cg->id)) {
-        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'cgroups to match as systemd services'", cg->id, cg->chart_title);
+        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'cgroups to match as systemd services'", cg->id, cg->chart_id);
         convert_cgroup_to_systemd_service(cg);
         return;
     }
 
     if (matches_enabled_cgroup_renames(cg->id)) {
-        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'run script to rename cgroups matching', will try to rename it", cg->id, cg->chart_title);
+        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') matches 'run script to rename cgroups matching', will try to rename it", cg->id, cg->chart_id);
         if (is_inside_k8s && k8s_is_container(cg->id)) {
             // it may take up to a minute for the K8s API to return data for the container
             // tested on AWS K8s cluster with 100% CPU utilization
@@ -2676,15 +2652,20 @@ static inline void discovery_process_first_time_seen_cgroup(struct cgroup *cg) {
 }
 
 static int discovery_is_cgroup_duplicate(struct cgroup *cg) {
-   // https://github.com/netdata/netdata/issues/797#issuecomment-241248884
-   struct cgroup *c;
-   for (c = discovered_cgroup_root; c; c = c->discovered_next) {
-       if (c != cg && c->enabled && c->hash_chart == cg->hash_chart && !strcmp(c->chart_id, cg->chart_id)) {
-           collector_error("CGROUP: chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.", cg->chart_id, c->id, cg->id);
-           return 1;
-       }
-   }
-   return 0;
+    // https://github.com/netdata/netdata/issues/797#issuecomment-241248884
+    struct cgroup *c;
+    for (c = discovered_cgroup_root; c; c = c->discovered_next) {
+        if (c != cg && c->enabled && (is_cgroup_systemd_service(c) == is_cgroup_systemd_service(cg)) &&
+            c->hash_chart_id == cg->hash_chart_id && !strcmp(c->chart_id, cg->chart_id)) {
+            collector_error(
+                "CGROUP: chart id '%s' already exists with id '%s' and is enabled and available. Disabling cgroup with id '%s'.",
+                cg->chart_id,
+                c->id,
+                cg->id);
+            return 1;
+        }
+    }
+    return 0;
 }
 
 static inline void discovery_process_cgroup(struct cgroup *cg) {
@@ -2720,17 +2701,25 @@ static inline void discovery_process_cgroup(struct cgroup *cg) {
     }
 
     if (is_cgroup_systemd_service(cg)) {
+        if (discovery_is_cgroup_duplicate(cg)) {
+            cg->enabled = 0;
+            cg->options |= CGROUP_OPTIONS_DISABLED_DUPLICATE;
+            return;
+        }
+        if (!cg->chart_labels)
+            cg->chart_labels = rrdlabels_create();
+        rrdlabels_add(cg->chart_labels, "service_name", cg->name, RRDLABEL_SRC_AUTO);
         cg->enabled = 1;
         return;
     }
 
-    if (!(cg->enabled = matches_enabled_cgroup_names(cg->chart_title))) {
-        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups names matching'", cg->id, cg->chart_title);
+    if (!(cg->enabled = matches_enabled_cgroup_names(cg->name))) {
+        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups names matching'", cg->id, cg->name);
         return;
     }
 
     if (!(cg->enabled = matches_enabled_cgroup_paths(cg->id))) {
-        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups matching'", cg->id, cg->chart_title);
+        netdata_log_debug(D_CGROUP, "cgroup '%s' (name '%s') disabled by 'enable by default cgroups matching'", cg->id, cg->name);
         return;
     }
 
@@ -2744,7 +2733,7 @@ static inline void discovery_process_cgroup(struct cgroup *cg) {
         cg->chart_labels = rrdlabels_create();
 
     if (!k8s_is_kubepod(cg)) {
-        rrdlabels_add(cg->chart_labels, "cgroup_name", cg->chart_id, RRDLABEL_SRC_AUTO);
+        rrdlabels_add(cg->chart_labels, "cgroup_name", cg->name, RRDLABEL_SRC_AUTO);
         if (!rrdlabels_exist(cg->chart_labels, "image"))
             rrdlabels_add(cg->chart_labels, "image", "", RRDLABEL_SRC_AUTO);
     }
@@ -2800,6 +2789,19 @@ static void cgroup_discovery_cleanup(void *ptr) {
     service_exits();
 }
 
+static inline char *cgroup_chart_type(char *buffer, struct cgroup *cg) {
+    if(buffer[0]) return buffer;
+
+    if (cg->chart_id[0] == '\0' || (cg->chart_id[0] == '/' && cg->chart_id[1] == '\0'))
+        strncpy(buffer, "cgroup_root", RRD_ID_LENGTH_MAX);
+    else if (is_cgroup_systemd_service(cg))
+        snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s%s", services_chart_id_prefix, cg->chart_id);
+    else
+        snprintfz(buffer, RRD_ID_LENGTH_MAX, "%s%s", cgroup_chart_id_prefix, cg->chart_id);
+
+    return buffer;
+}
+
 void cgroup_discovery_worker(void *ptr)
 {
     UNUSED(ptr);
@@ -2849,709 +2851,380 @@ void cgroup_discovery_worker(void *ptr)
 #define CHART_TITLE_MAX 300
 
 void update_systemd_services_charts(
-          int update_every
-        , int do_cpu
-        , int do_mem_usage
-        , int do_mem_detailed
-        , int do_mem_failcnt
-        , int do_swap_usage
-        , int do_io
-        , int do_io_ops
-        , int do_throttle_io
-        , int do_throttle_ops
-        , int do_queued_ops
-        , int do_merged_ops
-) {
-    static RRDSET
-        *st_cpu = NULL,
-        *st_mem_usage = NULL,
-        *st_mem_failcnt = NULL,
-        *st_swap_usage = NULL,
-
-        *st_mem_detailed_cache = NULL,
-        *st_mem_detailed_rss = NULL,
-        *st_mem_detailed_mapped = NULL,
-        *st_mem_detailed_writeback = NULL,
-        *st_mem_detailed_pgfault = NULL,
-        *st_mem_detailed_pgmajfault = NULL,
-        *st_mem_detailed_pgpgin = NULL,
-        *st_mem_detailed_pgpgout = NULL,
-
-        *st_io_read = NULL,
-        *st_io_serviced_read = NULL,
-        *st_throttle_io_read = NULL,
-        *st_throttle_ops_read = NULL,
-        *st_queued_ops_read = NULL,
-        *st_merged_ops_read = NULL,
-
-        *st_io_write = NULL,
-        *st_io_serviced_write = NULL,
-        *st_throttle_io_write = NULL,
-        *st_throttle_ops_write = NULL,
-        *st_queued_ops_write = NULL,
-        *st_merged_ops_write = NULL;
-
-    // create the charts
-
-    if (unlikely(do_cpu && !st_cpu)) {
-        char title[CHART_TITLE_MAX + 1];
-        snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (100%% = 1 core)");
-
-        st_cpu = rrdset_create_localhost(
-                "services"
-                , "cpu"
-                , NULL
-                , "cpu"
-                , "services.cpu"
-                , title
-                , "percentage"
-                , PLUGIN_CGROUPS_NAME
-                , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD
-                , update_every
-                , RRDSET_TYPE_STACKED
-        );
-    }
-
-    if (unlikely(do_mem_usage && !st_mem_usage)) {
-        st_mem_usage = rrdset_create_localhost(
-                "services"
-                , "mem_usage"
-                , NULL
-                , "mem"
-                , "services.mem_usage"
-                , "Systemd Services Used Memory"
-                , "MiB"
-                , PLUGIN_CGROUPS_NAME
-                , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 10
-                , update_every
-                , RRDSET_TYPE_STACKED
-        );
-    }
-
-    if(likely(do_mem_detailed)) {
-        if(unlikely(!st_mem_detailed_rss)) {
-            st_mem_detailed_rss = rrdset_create_localhost(
-                    "services"
-                    , "mem_rss"
-                    , NULL
-                    , "mem"
-                    , "services.mem_rss"
-                    , "Systemd Services RSS Memory"
-                    , "MiB"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 20
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_mem_detailed_mapped)) {
-            st_mem_detailed_mapped = rrdset_create_localhost(
-                    "services"
-                    , "mem_mapped"
-                    , NULL
-                    , "mem"
-                    , "services.mem_mapped"
-                    , "Systemd Services Mapped Memory"
-                    , "MiB"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 30
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_mem_detailed_cache)) {
-            st_mem_detailed_cache = rrdset_create_localhost(
-                    "services"
-                    , "mem_cache"
-                    , NULL
-                    , "mem"
-                    , "services.mem_cache"
-                    , "Systemd Services Cache Memory"
-                    , "MiB"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 40
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_mem_detailed_writeback)) {
-            st_mem_detailed_writeback = rrdset_create_localhost(
-                    "services"
-                    , "mem_writeback"
-                    , NULL
-                    , "mem"
-                    , "services.mem_writeback"
-                    , "Systemd Services Writeback Memory"
-                    , "MiB"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 50
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-        }
-
-        if(unlikely(!st_mem_detailed_pgfault)) {
-            st_mem_detailed_pgfault = rrdset_create_localhost(
-                    "services"
-                    , "mem_pgfault"
-                    , NULL
-                    , "mem"
-                    , "services.mem_pgfault"
-                    , "Systemd Services Memory Minor Page Faults"
-                    , "MiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 60
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_mem_detailed_pgmajfault)) {
-            st_mem_detailed_pgmajfault = rrdset_create_localhost(
-                    "services"
-                    , "mem_pgmajfault"
-                    , NULL
-                    , "mem"
-                    , "services.mem_pgmajfault"
-                    , "Systemd Services Memory Major Page Faults"
-                    , "MiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 70
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_mem_detailed_pgpgin)) {
-            st_mem_detailed_pgpgin = rrdset_create_localhost(
-                    "services"
-                    , "mem_pgpgin"
-                    , NULL
-                    , "mem"
-                    , "services.mem_pgpgin"
-                    , "Systemd Services Memory Charging Activity"
-                    , "MiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 80
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-        }
-
-        if(unlikely(!st_mem_detailed_pgpgout)) {
-            st_mem_detailed_pgpgout = rrdset_create_localhost(
-                    "services"
-                    , "mem_pgpgout"
-                    , NULL
-                    , "mem"
-                    , "services.mem_pgpgout"
-                    , "Systemd Services Memory Uncharging Activity"
-                    , "MiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 90
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-    }
-
-    if(unlikely(do_mem_failcnt && !st_mem_failcnt)) {
-        st_mem_failcnt = rrdset_create_localhost(
-                "services"
-                , "mem_failcnt"
-                , NULL
-                , "mem"
-                , "services.mem_failcnt"
-                , "Systemd Services Memory Limit Failures"
-                , "failures"
-                , PLUGIN_CGROUPS_NAME
-                , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 110
-                , update_every
-                , RRDSET_TYPE_STACKED
-        );
-    }
-
-    if (do_swap_usage && !st_swap_usage) {
-        st_swap_usage = rrdset_create_localhost(
-                "services"
-                , "swap_usage"
-                , NULL
-                , "swap"
-                , "services.swap_usage"
-                , "Systemd Services Swap Memory Used"
-                , "MiB"
-                , PLUGIN_CGROUPS_NAME
-                , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 100
-                , update_every
-                , RRDSET_TYPE_STACKED
-        );
-    }
-
-    if(likely(do_io)) {
-        if(unlikely(!st_io_read)) {
-            st_io_read = rrdset_create_localhost(
-                    "services"
-                    , "io_read"
-                    , NULL
-                    , "disk"
-                    , "services.io_read"
-                    , "Systemd Services Disk Read Bandwidth"
-                    , "KiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 120
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_io_write)) {
-            st_io_write = rrdset_create_localhost(
-                    "services"
-                    , "io_write"
-                    , NULL
-                    , "disk"
-                    , "services.io_write"
-                    , "Systemd Services Disk Write Bandwidth"
-                    , "KiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 130
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-    }
-
-    if(likely(do_io_ops)) {
-        if(unlikely(!st_io_serviced_read)) {
-            st_io_serviced_read = rrdset_create_localhost(
-                    "services"
-                    , "io_ops_read"
-                    , NULL
-                    , "disk"
-                    , "services.io_ops_read"
-                    , "Systemd Services Disk Read Operations"
-                    , "operations/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 140
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-
-        if(unlikely(!st_io_serviced_write)) {
-            st_io_serviced_write = rrdset_create_localhost(
-                    "services"
-                    , "io_ops_write"
-                    , NULL
-                    , "disk"
-                    , "services.io_ops_write"
-                    , "Systemd Services Disk Write Operations"
-                    , "operations/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 150
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-        }
-    }
-
-    if(likely(do_throttle_io)) {
-        if(unlikely(!st_throttle_io_read)) {
-
-            st_throttle_io_read = rrdset_create_localhost(
-                    "services"
-                    , "throttle_io_read"
-                    , NULL
-                    , "disk"
-                    , "services.throttle_io_read"
-                    , "Systemd Services Throttle Disk Read Bandwidth"
-                    , "KiB/s"
-                    , PLUGIN_CGROUPS_NAME
-                    , PLUGIN_CGROUPS_MODULE_SYSTEMD_NAME
-                    , NETDATA_CHART_PRIO_CGROUPS_SYSTEMD + 160
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-        }
-
-        if(unlikely(!st_throttle_io_write)) {
-            st_throttle_io_write = rrdset_create_localhost(
-                    "services"
-                    , "throttle_io_write"
-                    , NULL
-                    , "disk"
-                    , "services.throttle_io_write"
-