diff options
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | collectors/ebpf.plugin/Makefile.am | 1 | ||||
-rw-r--r-- | collectors/ebpf.plugin/README.md | 3 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf.c | 27 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf.d.conf | 2 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf.d/softirq.conf | 8 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf.h | 4 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf_apps.h | 3 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf_hardirq.c | 2 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf_softirq.c | 275 | ||||
-rw-r--r-- | collectors/ebpf.plugin/ebpf_softirq.h | 35 | ||||
-rw-r--r-- | packaging/ebpf.checksums | 6 | ||||
-rw-r--r-- | packaging/ebpf.version | 2 | ||||
-rw-r--r-- | web/gui/dashboard_info.js | 6 |
15 files changed, 369 insertions, 9 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index ea47ec20c9..e73fc22e95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -504,6 +504,8 @@ set(EBPF_PROCESS_PLUGIN_FILES collectors/ebpf.plugin/ebpf_process.h collectors/ebpf.plugin/ebpf_socket.c collectors/ebpf.plugin/ebpf_socket.h + collectors/ebpf.plugin/ebpf_softirq.c + collectors/ebpf.plugin/ebpf_softirq.h collectors/ebpf.plugin/ebpf_sync.c collectors/ebpf.plugin/ebpf_sync.h collectors/ebpf.plugin/ebpf_swap.c diff --git a/Makefile.am b/Makefile.am index 1f7df5dad4..f47fea1ea3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -308,6 +308,8 @@ EBPF_PLUGIN_FILES = \ collectors/ebpf.plugin/ebpf_process.h \ collectors/ebpf.plugin/ebpf_socket.c \ collectors/ebpf.plugin/ebpf_socket.h \ + collectors/ebpf.plugin/ebpf_softirq.c \ + collectors/ebpf.plugin/ebpf_softirq.h \ collectors/ebpf.plugin/ebpf_sync.c \ collectors/ebpf.plugin/ebpf_sync.h \ collectors/ebpf.plugin/ebpf_swap.c \ diff --git a/collectors/ebpf.plugin/Makefile.am b/collectors/ebpf.plugin/Makefile.am index 5a659f47d3..67554ba73b 100644 --- a/collectors/ebpf.plugin/Makefile.am +++ b/collectors/ebpf.plugin/Makefile.am @@ -41,6 +41,7 @@ dist_ebpfconfig_DATA = \ ebpf.d/mount.conf \ ebpf.d/network.conf \ ebpf.d/process.conf \ + ebpf.d/softirq.conf \ ebpf.d/sync.conf \ ebpf.d/swap.conf \ ebpf.d/vfs.conf \ diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md index 79f6b54f97..27071064ba 100644 --- a/collectors/ebpf.plugin/README.md +++ b/collectors/ebpf.plugin/README.md @@ -320,6 +320,8 @@ The eBPF collector enables and runs the following eBPF programs by default: When in `return` mode, it also creates charts showing errors when these operations are executed. - `hardirq`: This eBPF program creates charts that show information about time spent servicing individual hardware interrupt requests (hard IRQs). +- `softirq`: This eBPF program creates charts that show information about + time spent servicing individual software interrupt requests (soft IRQs). You can also enable the following eBPF programs: - `cachestat`: Netdata's eBPF data collector creates charts about the memory page cache. When the integration with @@ -355,6 +357,7 @@ The following configuration files are available: - `process.conf`: Configuration for the `process` thread. - `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and also lets you specify which network the eBPF collector monitors. +- `softirq.conf`: Configuration for the `softirq` thread. - `sync.conf`: Configuration for the `sync` thread. - `vfs.conf`: Configuration for the `vfs` thread. diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c index 7e85dc14ca..5df96487a5 100644 --- a/collectors/ebpf.plugin/ebpf.c +++ b/collectors/ebpf.plugin/ebpf.c @@ -135,6 +135,11 @@ ebpf_module_t ebpf_modules[] = { .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &hardirq_config, .config_file = NETDATA_HARDIRQ_CONFIG_FILE}, + { .thread_name = "softirq", .config_name = "softirq", .enabled = 0, .start_routine = ebpf_softirq_thread, + .update_time = 1, .global_charts = 1, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, + .optional = 0, .apps_routine = NULL, .maps = NULL, + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config, + .config_file = NETDATA_SOFTIRQ_CONFIG_FILE}, { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 0, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL, @@ -729,6 +734,8 @@ void ebpf_print_help() "\n" " --return or -r Run the collector in return mode.\n" "\n", + " --softirq or -t Enable chart related to soft IRQ latency.\n" + "\n" " --sync or -s Enable chart related to sync run time.\n" "\n" " --swap or -w Enable chart related to swap run time.\n" @@ -1154,6 +1161,13 @@ static void read_collector_values(int *disable_apps) started++; } + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "softirq", + CONFIG_BOOLEAN_YES); + if (enabled) { + ebpf_enable_chart(EBPF_MODULE_SOFTIRQ_IDX, *disable_apps); + started++; + } + if (!started){ ebpf_enable_all_charts(*disable_apps); // Read network viewer section @@ -1246,6 +1260,7 @@ static void parse_args(int argc, char **argv) {"net", no_argument, 0, 'n' }, {"process", no_argument, 0, 'p' }, {"return", no_argument, 0, 'r' }, + {"softirq", no_argument, 0, 't' }, {"sync", no_argument, 0, 's' }, {"swap", no_argument, 0, 'w' }, {"vfs", no_argument, 0, 'f' }, @@ -1263,7 +1278,7 @@ static void parse_args(int argc, char **argv) } while (1) { - int c = getopt_long(argc, argv, "hvgacdkieqmnprswf", long_options, &option_index); + int c = getopt_long(argc, argv, "hvgacdkieqmnprtswf", long_options, &option_index); if (c == -1) break; @@ -1374,6 +1389,14 @@ static void parse_args(int argc, char **argv) #endif break; } + case 't': { + enabled = 1; + ebpf_enable_chart(EBPF_MODULE_SOFTIRQ_IDX, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF enabling \"softirq\" chart, because it was started with the option \"--softirq\" or \"-t\"."); +#endif + break; + } case 's': { enabled = 1; ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, disable_apps); @@ -1680,6 +1703,8 @@ int main(int argc, char **argv) NULL, NULL, ebpf_modules[EBPF_MODULE_FD_IDX].start_routine}, {"EBPF HARDIRQ" , NULL, NULL, 1, NULL, NULL, ebpf_modules[EBPF_MODULE_HARDIRQ_IDX].start_routine}, + {"EBPF SOFTIRQ" , NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_SOFTIRQ_IDX].start_routine}, {NULL , NULL, NULL, 0, NULL, NULL, NULL} }; diff --git a/collectors/ebpf.plugin/ebpf.d.conf b/collectors/ebpf.plugin/ebpf.d.conf index 4062e22c5f..a047e0e56e 100644 --- a/collectors/ebpf.plugin/ebpf.d.conf +++ b/collectors/ebpf.plugin/ebpf.d.conf @@ -35,6 +35,7 @@ # `process` : This eBPF program creates charts that show information about process life. # `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the # bandwidth consumed by each. +# `softirq` : Monitor latency of serving software interrupt requests (soft IRQs). # `sync` : Montitor calls for syscall sync(2). # `swap` : Monitor calls for internal swap functions. # `vfs` : This eBPF program creates charts that show information about process VFS IO, VFS file manipulation and @@ -49,6 +50,7 @@ mount = yes process = yes socket = yes + softirq = yes sync = yes swap = no vfs = yes diff --git a/collectors/ebpf.plugin/ebpf.d/softirq.conf b/collectors/ebpf.plugin/ebpf.d/softirq.conf new file mode 100644 index 0000000000..976991a086 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/softirq.conf @@ -0,0 +1,8 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +[global] + ebpf load mode = entry + update every = 1 diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h index 2fe241610c..8bbd7124f9 100644 --- a/collectors/ebpf.plugin/ebpf.h +++ b/collectors/ebpf.plugin/ebpf.h @@ -85,7 +85,8 @@ enum ebpf_module_indexes { EBPF_MODULE_DISK_IDX, EBPF_MODULE_MOUNT_IDX, EBPF_MODULE_FD_IDX, - EBPF_MODULE_HARDIRQ_IDX + EBPF_MODULE_HARDIRQ_IDX, + EBPF_MODULE_SOFTIRQ_IDX }; typedef struct ebpf_tracepoint { @@ -223,6 +224,7 @@ extern uint32_t ebpf_enable_tracepoints(ebpf_tracepoint_t *tps); #define EBPF_COMMON_DIMENSION_DIFFERENCE "difference" #define EBPF_COMMON_DIMENSION_PACKETS "packets" #define EBPF_COMMON_DIMENSION_FILES "files" +#define EBPF_COMMON_DIMENSION_MILLISECONDS "milliseconds" // Common variables extern int debug_enabled; diff --git a/collectors/ebpf.plugin/ebpf_apps.h b/collectors/ebpf.plugin/ebpf_apps.h index 3f53afc3b0..acc1e8449d 100644 --- a/collectors/ebpf.plugin/ebpf_apps.h +++ b/collectors/ebpf.plugin/ebpf_apps.h @@ -23,12 +23,13 @@ #include "ebpf_disk.h" #include "ebpf_fd.h" #include "ebpf_filesystem.h" +#include "ebpf_hardirq.h" #include "ebpf_cachestat.h" #include "ebpf_mount.h" +#include "ebpf_softirq.h" #include "ebpf_sync.h" #include "ebpf_swap.h" #include "ebpf_vfs.h" -#include "ebpf_hardirq.h" #define MAX_COMPARE_NAME 100 #define MAX_NAME 100 diff --git a/collectors/ebpf.plugin/ebpf_hardirq.c b/collectors/ebpf.plugin/ebpf_hardirq.c index e95091e691..29ece2ddf2 100644 --- a/collectors/ebpf.plugin/ebpf_hardirq.c +++ b/collectors/ebpf.plugin/ebpf_hardirq.c @@ -344,7 +344,7 @@ static void hardirq_create_charts() NETDATA_EBPF_SYSTEM_GROUP, "hardirq_latency", "Hardware IRQ latency", - "milliseconds", + EBPF_COMMON_DIMENSION_MILLISECONDS, "interrupts", NULL, NETDATA_EBPF_CHART_TYPE_STACKED, diff --git a/collectors/ebpf.plugin/ebpf_softirq.c b/collectors/ebpf.plugin/ebpf_softirq.c new file mode 100644 index 0000000000..ba700cd120 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_softirq.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ebpf.h" +#include "ebpf_softirq.h" + +struct config softirq_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + +#define SOFTIRQ_MAP_LATENCY 0 +static ebpf_local_maps_t softirq_maps[] = { + { + .name = "tbl_softirq", + .internal_input = NETDATA_SOFTIRQ_MAX_IRQS, + .user_input = 0, + .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + }, + /* end */ + { + .name = NULL, + .internal_input = 0, + .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + } +}; + +static ebpf_data_t softirq_data; + +#define SOFTIRQ_TP_CLASS_IRQ "irq" +static ebpf_tracepoint_t softirq_tracepoints[] = { + {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_entry"}, + {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_exit"}, + /* end */ + {.enabled = false, .class = NULL, .event = NULL} +}; + +// these must be in the order defined by the kernel: +// https://elixir.bootlin.com/linux/v5.12.19/source/include/trace/events/irq.h#L13 +static softirq_val_t softirq_vals[] = { + {.name = "HI", .latency = 0}, + {.name = "TIMER", .latency = 0}, + {.name = "NET_TX", .latency = 0}, + {.name = "NET_RX", .latency = 0}, + {.name = "BLOCK", .latency = 0}, + {.name = "IRQ_POLL", .latency = 0}, + {.name = "TASKLET", .latency = 0}, + {.name = "SCHED", .latency = 0}, + {.name = "HRTIMER", .latency = 0}, + {.name = "RCU", .latency = 0}, +}; + +// tmp store for soft IRQ values we get from a per-CPU eBPF map. +static softirq_ebpf_val_t *softirq_ebpf_vals = NULL; + +static struct bpf_link **probe_links = NULL; +static struct bpf_object *objects = NULL; + +static int read_thread_closed = 1; + +static struct netdata_static_thread softirq_threads = {"SOFTIRQ KERNEL", + NULL, NULL, 1, NULL, + NULL, NULL }; + +/** + * Clean up the main thread. + * + * @param ptr thread data. + */ +static void softirq_cleanup(void *ptr) +{ + for (int i = 0; softirq_tracepoints[i].class != NULL; i++) { + ebpf_disable_tracepoint(&softirq_tracepoints[i]); + } + + ebpf_module_t *em = (ebpf_module_t *)ptr; + if (!em->enabled) { + return; + } + + heartbeat_t hb; + heartbeat_init(&hb); + uint32_t tick = 1 * USEC_PER_MS; + while (!read_thread_closed) { + usec_t dt = heartbeat_next(&hb, tick); + UNUSED(dt); + } + + freez(softirq_ebpf_vals); + freez(softirq_threads.thread); + + if (probe_links) { + struct bpf_program *prog; + size_t i = 0 ; + bpf_object__for_each_program(prog, objects) { + bpf_link__destroy(probe_links[i]); + i++; + } + bpf_object__close(objects); + } +} + +/***************************************************************** + * MAIN LOOP + *****************************************************************/ + +static void softirq_read_latency_map() +{ + int fd = softirq_maps[SOFTIRQ_MAP_LATENCY].map_fd; + int i; + for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) { + int test = bpf_map_lookup_elem(fd, &i, softirq_ebpf_vals); + if (unlikely(test < 0)) { + continue; + } + + uint64_t total_latency = 0; + int cpu_i; + int end = ebpf_nprocs; + for (cpu_i = 0; cpu_i < end; cpu_i++) { + total_latency += softirq_ebpf_vals[cpu_i].latency/1000; + } + + softirq_vals[i].latency = total_latency; + } +} + +/** + * Read eBPF maps for soft IRQ. + */ +static void *softirq_reader(void *ptr) +{ + read_thread_closed = 0; + + heartbeat_t hb; + heartbeat_init(&hb); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + + usec_t step = NETDATA_SOFTIRQ_SLEEP_MS * em->update_time; + while (!close_ebpf_plugin) { + usec_t dt = heartbeat_next(&hb, step); + UNUSED(dt); + + softirq_read_latency_map(); + } + + read_thread_closed = 1; + return NULL; +} + +static void softirq_create_charts() +{ + ebpf_create_chart( + NETDATA_EBPF_SYSTEM_GROUP, + "softirq_latency", + "Software IRQ latency", + EBPF_COMMON_DIMENSION_MILLISECONDS, + "softirqs", + NULL, + NETDATA_EBPF_CHART_TYPE_STACKED, + NETDATA_CHART_PRIO_SYSTEM_SOFTIRQS+1, + NULL, NULL, 0, + NETDATA_EBPF_MODULE_NAME_SOFTIRQ + ); + + fflush(stdout); +} + +static void softirq_create_dims() +{ + uint32_t i; + for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) { + ebpf_write_global_dimension( + softirq_vals[i].name, softirq_vals[i].name, + ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX] + ); + } +} + +static inline void softirq_write_dims() +{ + uint32_t i; + for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) { + write_chart_dimension(softirq_vals[i].name, softirq_vals[i].latency); + } +} + +/** +* Main loop for this collector. +*/ +static void softirq_collector(ebpf_module_t *em) +{ + softirq_ebpf_vals = callocz(ebpf_nprocs, sizeof(softirq_ebpf_val_t)); + + // create reader thread. + softirq_threads.thread = mallocz(sizeof(netdata_thread_t)); + softirq_threads.start_routine = softirq_reader; + netdata_thread_create( + softirq_threads.thread, + softirq_threads.name, + NETDATA_THREAD_OPTION_JOINABLE, + softirq_reader, + em + ); + + // create chart and static dims. + pthread_mutex_lock(&lock); + softirq_create_charts(); + softirq_create_dims(); + pthread_mutex_unlock(&lock); + + // loop and read from published data until ebpf plugin is closed. + while (!close_ebpf_plugin) { + pthread_mutex_lock(&collect_data_mutex); + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex); + pthread_mutex_lock(&lock); + + // write dims now for all hitherto discovered IRQs. + write_begin_chart(NETDATA_EBPF_SYSTEM_GROUP, "softirq_latency"); + softirq_write_dims(); + write_end_chart(); + + pthread_mutex_unlock(&lock); + pthread_mutex_unlock(&collect_data_mutex); + } +} + +/***************************************************************** + * EBPF SOFTIRQ THREAD + *****************************************************************/ + +/** + * Soft IRQ latency thread. + * + * @param ptr a `ebpf_module_t *`. + * @return always NULL. + */ +void *ebpf_softirq_thread(void *ptr) +{ + netdata_thread_cleanup_push(softirq_cleanup, ptr); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + em->maps = softirq_maps; + + fill_ebpf_data(&softirq_data); + + if (!em->enabled) { + goto endsoftirq; + } + + if (ebpf_update_kernel(&softirq_data)) { + goto endsoftirq; + } + + if (ebpf_enable_tracepoints(softirq_tracepoints) == 0) { + em->enabled = CONFIG_BOOLEAN_NO; + goto endsoftirq; + } + + probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, softirq_data.map_fd); + if (!probe_links) { + goto endsoftirq; + } + + softirq_collector(em); + +endsoftirq: + netdata_thread_cleanup_pop(1); + + return NULL; +} diff --git a/collectors/ebpf.plugin/ebpf_softirq.h b/collectors/ebpf.plugin/ebpf_softirq.h new file mode 100644 index 0000000000..c556e1fd2b --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_softirq.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EBPF_SOFTIRQ_H +#define NETDATA_EBPF_SOFTIRQ_H 1 + +/***************************************************************** + * copied from kernel-collectors repo, with modifications needed + * for inclusion here. + *****************************************************************/ + +#define NETDATA_SOFTIRQ_MAX_IRQS 10 + +typedef struct softirq_ebpf_val { + uint64_t latency; + uint64_t ts; +} softirq_ebpf_val_t; + +/***************************************************************** + * below this is eBPF plugin-specific code. + *****************************************************************/ + +#define NETDATA_EBPF_MODULE_NAME_SOFTIRQ "softirq" +#define NETDATA_SOFTIRQ_SLEEP_MS 650000ULL +#define NETDATA_SOFTIRQ_CONFIG_FILE "softirq.conf" + +typedef struct sofirq_val { + uint64_t latency; + char *name; +} softirq_val_t; + +extern struct config softirq_config; +extern void *ebpf_softirq_thread(void *ptr); +extern void ebpf_softirq_create_apps_charts(struct ebpf_module *em, void *ptr); + +#endif /* NETDATA_EBPF_SOFTIRQ_H */ diff --git a/packaging/ebpf.checksums b/packaging/ebpf.checksums index c6fce7cb65..f8ff738675 100644 --- a/packaging/ebpf.checksums +++ b/packaging/ebpf.checksums @@ -1,4 +1,4 @@ -a56eece7b09b795a3a75e317baff0bc46b3207a0a5535c179dc4bce6a144de36 netdata-kernel-collector-glibc-v0.7.8.1.tar.xz -5e98ed15ab8ba667065f17f9722f4cf86f2932e454015c2a5c169ccb950bbb33 netdata-kernel-collector-musl-v0.7.8.1.tar.xz -d93dff80d0b0e41478f734f44edfdb1bf1cfbd57db4c9f4fc34d703f436a77a6 netdata-kernel-collector-static-v0.7.8.1.tar.xz +cb7079f209d34b00a4d8e1e7a228bc37b736d539b36a0649dcfda9743c64ddff netdata-kernel-collector-glibc-v0.7.9.tar.xz +7832a73d6d4eeae92f38fc47fbdf77fa6c3b7442e1b3fd6cfc2b1a1df06d6895 netdata-kernel-collector-musl-v0.7.9.tar.xz +620c459819363f207c8f91aec817be222859e523d3972bc5080737475b93ad40 netdata-kernel-collector-static-v0.7.9.tar.xz diff --git a/packaging/ebpf.version b/packaging/ebpf.version index 102a1331a3..5d0bf3f289 100644 --- a/packaging/ebpf.version +++ b/packaging/ebpf.version @@ -1,2 +1,2 @@ -v0.7.8.1 +v0.7.9 diff --git a/web/gui/dashboard_info.js b/web/gui/dashboard_info.js index e2e03c9e58..eab15c8212 100644 --- a/web/gui/dashboard_info.js +++ b/web/gui/dashboard_info.js @@ -953,13 +953,17 @@ netdataDashboard.context = { }, 'system.hardirq_latency': { - info: 'Total time spent servicing hardware interrupts.' + info: 'Total time spent servicing hardware interrupts. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/hardirqs_example.txt" target="_blank">hardirqs</a> from BCC tools.' }, 'system.softirqs': { info: 'CPU softirqs in detail. At the <a href="#menu_cpu">CPUs</a> section, softirqs are analyzed per CPU core.' }, + 'system.softirq_latency': { + info: 'Total time spent servicing software interrupts. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/softirqs_example.txt" target="_blank">softirqs</a> from BCC tools.' + }, + 'system.processes': { info: 'System processes. <b>Running</b> are the processes in the CPU. <b>Blocked</b> are processes that are willing to enter the CPU, but they cannot, e.g. because they wait for disk activity.' }, |