From 37686c8d682dc5235f8c1b790f484242fc194401 Mon Sep 17 00:00:00 2001 From: thiagoftsm Date: Fri, 2 Jul 2021 12:28:50 +0000 Subject: Ebpf disk latency (#11276) Add disk monitoring independent of filesystem. --- CMakeLists.txt | 2 + Makefile.am | 2 + collectors/all.h | 1 + collectors/ebpf.plugin/Makefile.am | 1 + collectors/ebpf.plugin/README.md | 2 + collectors/ebpf.plugin/ebpf.c | 50 ++ collectors/ebpf.plugin/ebpf.d.conf | 3 + collectors/ebpf.plugin/ebpf.d/disk.conf | 13 + collectors/ebpf.plugin/ebpf.h | 4 +- collectors/ebpf.plugin/ebpf_apps.h | 1 + collectors/ebpf.plugin/ebpf_disk.c | 837 +++++++++++++++++++++++++++++++ collectors/ebpf.plugin/ebpf_disk.h | 72 +++ collectors/ebpf.plugin/ebpf_filesystem.c | 63 +-- collectors/ebpf.plugin/ebpf_filesystem.h | 9 - libnetdata/ebpf/ebpf.c | 111 ++++ libnetdata/ebpf/ebpf.h | 19 + packaging/ebpf.checksums | 6 +- packaging/ebpf.version | 2 +- web/gui/dashboard_info.js | 4 + 19 files changed, 1145 insertions(+), 57 deletions(-) create mode 100644 collectors/ebpf.plugin/ebpf.d/disk.conf create mode 100644 collectors/ebpf.plugin/ebpf_disk.c create mode 100644 collectors/ebpf.plugin/ebpf_disk.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c5d4bc8f52..2a50ff51d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,6 +490,8 @@ set(EBPF_PROCESS_PLUGIN_FILES collectors/ebpf.plugin/ebpf_cachestat.h collectors/ebpf.plugin/ebpf_dcstat.c collectors/ebpf.plugin/ebpf_dcstat.h + collectors/ebpf.plugin/ebpf_disk.c + collectors/ebpf.plugin/ebpf_disk.h collectors/ebpf.plugin/ebpf_filesystem.c collectors/ebpf.plugin/ebpf_filesystem.h collectors/ebpf.plugin/ebpf_process.c diff --git a/Makefile.am b/Makefile.am index 1c98e30b6a..3c47b62e6d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -289,6 +289,8 @@ EBPF_PLUGIN_FILES = \ collectors/ebpf.plugin/ebpf_cachestat.h \ collectors/ebpf.plugin/ebpf_dcstat.c \ collectors/ebpf.plugin/ebpf_dcstat.h \ + collectors/ebpf.plugin/ebpf_disk.c \ + collectors/ebpf.plugin/ebpf_disk.h \ collectors/ebpf.plugin/ebpf_filesystem.c \ collectors/ebpf.plugin/ebpf_filesystem.h \ collectors/ebpf.plugin/ebpf_process.c \ diff --git a/collectors/all.h b/collectors/all.h index b0e09f5364..41bed0592f 100644 --- a/collectors/all.h +++ b/collectors/all.h @@ -116,6 +116,7 @@ #define NETDATA_CHART_PRIO_DISK_SVCTM 2070 #define NETDATA_CHART_PRIO_DISK_MOPS 2080 #define NETDATA_CHART_PRIO_DISK_IOTIME 2090 +#define NETDATA_CHART_PRIO_DISK_LATENCY 2095 #define NETDATA_CHART_PRIO_BCACHE_CACHE_ALLOC 2120 #define NETDATA_CHART_PRIO_BCACHE_HIT_RATIO 2120 #define NETDATA_CHART_PRIO_BCACHE_RATES 2121 diff --git a/collectors/ebpf.plugin/Makefile.am b/collectors/ebpf.plugin/Makefile.am index 2b73ed2ddc..30552dd006 100644 --- a/collectors/ebpf.plugin/Makefile.am +++ b/collectors/ebpf.plugin/Makefile.am @@ -34,6 +34,7 @@ dist_ebpfconfig_DATA = \ ebpf.d/ebpf_kernel_reject_list.txt \ ebpf.d/cachestat.conf \ ebpf.d/dcstat.conf \ + ebpf.d/disk.conf \ ebpf.d/filesystem.conf \ ebpf.d/network.conf \ ebpf.d/process.conf \ diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md index 839c6ab721..3ab1a8ea17 100644 --- a/collectors/ebpf.plugin/README.md +++ b/collectors/ebpf.plugin/README.md @@ -216,6 +216,7 @@ The eBPF collector enables and runs the following eBPF programs by default: - `dcstat` : This eBPF program creates charts that show information about file access using directory cache. It appends `kprobes` for `lookup_fast()` and `d_lookup()` to identify if files are inside directory cache, outside and files are not found. +- `disk` : This eBPF program creates charts that show information about disk latency independent of filesystem. - `filesystem`: This eBPF program creates charts that show latency information for selected filesystem. - `process`: This eBPF program creates charts that show information about process creation, calls to open files. When in `return` mode, it also creates charts showing errors when these operations are executed. @@ -240,6 +241,7 @@ The following configuration files are available: - `cachestat.conf`: Configuration for the `cachestat` thread. - `dcstat.conf`: Configuration for the `dcstat` thread. +- `disk.conf`: Configuration for the `disk` thread. - `filesystem.conf`: Configuration for the `filesystem` thread. - `process.conf`: Configuration for the `process` thread. - `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c index d419861904..9807cda2a2 100644 --- a/collectors/ebpf.plugin/ebpf.c +++ b/collectors/ebpf.plugin/ebpf.c @@ -115,6 +115,11 @@ ebpf_module_t ebpf_modules[] = { .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fs_config, .config_file = NETDATA_SYNC_CONFIG_FILE}, + { .thread_name = "disk", .config_name = "disk", .enabled = 0, .start_routine = ebpf_disk_thread, + .update_time = 1, .global_charts = 1, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, + .optional = 0, .apps_routine = NULL, .maps = NULL, + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &disk_config, + .config_file = NETDATA_SYNC_CONFIG_FILE}, { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 0, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL, @@ -509,6 +514,31 @@ void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family } } +/** + * Call the necessary functions to create a name. + * + * @param family family name + * @param name chart name + * @param hist0 histogram values + * @param dimensions dimension values. + * @param end number of bins that will be sent to Netdata. + * + * @return It returns a variable tha maps the charts that did not have zero values. + */ +void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end) +{ + write_begin_chart(family, name); + + uint32_t i; + for (i = 0; i < end; i++) { + write_chart_dimension(dimensions[i], (long long) hist[i]); + } + + write_end_chart(); + + fflush(stdout); +} + /***************************************************************** * * FUNCTIONS TO DEFINE OPTIONS @@ -658,6 +688,8 @@ void ebpf_print_help() "\n" " --dcstat or -d Enable charts related to directory cache.\n" "\n" + " --disk or -k Enable charts related to disk monitoring.\n" + "\n" " --filesystem or -i Enable chart related to filesystem run time.\n" "\n" " --net or -n Enable network viewer charts.\n" @@ -982,6 +1014,13 @@ static void read_collector_values(int *disable_apps) started++; } + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "disk", + CONFIG_BOOLEAN_NO); + if (enabled) { + ebpf_enable_chart(EBPF_MODULE_DISK_IDX, *disable_apps); + started++; + } + if (!started){ ebpf_enable_all_charts(*disable_apps); // Read network viewer section @@ -1066,6 +1105,7 @@ static void parse_args(int argc, char **argv) {"all", no_argument, 0, 'a' }, {"cachestat", no_argument, 0, 'c' }, {"dcstat", no_argument, 0, 'd' }, + {"disk", no_argument, 0, 'k' }, {"filesystem", no_argument, 0, 'i' }, {"net", no_argument, 0, 'n' }, {"process", no_argument, 0, 'p' }, @@ -1139,6 +1179,14 @@ static void parse_args(int argc, char **argv) ebpf_enable_chart(EBPF_MODULE_FILESYSTEM_IDX, disable_apps); #ifdef NETDATA_INTERNAL_CHECKS info("EBPF enabling \"filesystem\" chart, because it was started with the option \"--filesystem\" or \"-i\"."); +#endif + break; + } + case 'k': { + enabled = 1; + ebpf_enable_chart(EBPF_MODULE_DISK_IDX, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF enabling \"disk\" chart, because it was started with the option \"--disk\" or \"-k\"."); #endif break; } @@ -1464,6 +1512,8 @@ int main(int argc, char **argv) NULL, NULL, ebpf_modules[EBPF_MODULE_VFS_IDX].start_routine}, {"EBPF FILESYSTEM" , NULL, NULL, 1, NULL, NULL, ebpf_modules[EBPF_MODULE_FILESYSTEM_IDX].start_routine}, + {"EBPF DISK" , NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_DISK_IDX].start_routine}, {NULL , NULL, NULL, 0, NULL, NULL, NULL} }; diff --git a/collectors/ebpf.plugin/ebpf.d.conf b/collectors/ebpf.plugin/ebpf.d.conf index 6bc771e9a2..15512e76fd 100644 --- a/collectors/ebpf.plugin/ebpf.d.conf +++ b/collectors/ebpf.plugin/ebpf.d.conf @@ -26,6 +26,8 @@ # The eBPF collector enables and runs the following eBPF programs by default: # # `cachestat` : Make charts for kernel functions related to page cache. +# `dcstat` : Make charts for kernel functions related to directory cache. +# `disk` : Monitor I/O latencies for disks # `filesystem`: Monitor calls for functions used to manipulate specific filesystems # `process` : This eBPF program creates charts that show information about process creation, and file manipulation. # `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the @@ -37,6 +39,7 @@ [ebpf programs] cachestat = no dcstat = no + disk = no filesystem = no process = yes socket = yes diff --git a/collectors/ebpf.plugin/ebpf.d/disk.conf b/collectors/ebpf.plugin/ebpf.d/disk.conf new file mode 100644 index 0000000000..6ba18477c4 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/disk.conf @@ -0,0 +1,13 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`. +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to +# 'no'. +# +[global] + ebpf load mode = entry + update every = 2 + diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h index 7036385add..9c099815a3 100644 --- a/collectors/ebpf.plugin/ebpf.h +++ b/collectors/ebpf.plugin/ebpf.h @@ -81,7 +81,8 @@ enum ebpf_module_indexes { EBPF_MODULE_DCSTAT_IDX, EBPF_MODULE_SWAP_IDX, EBPF_MODULE_VFS_IDX, - EBPF_MODULE_FILESYSTEM_IDX + EBPF_MODULE_FILESYSTEM_IDX, + EBPF_MODULE_DISK_IDX }; // Copied from musl header @@ -226,6 +227,7 @@ extern collected_number get_value_from_structure(char *basis, size_t offset); extern void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em); extern void ebpf_write_chart_obsolete(char *type, char *id, char *title, char *units, char *family, char *charttype, char *context, int order); +extern void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end); #define EBPF_MAX_SYNCHRONIZATION_TIME 300 diff --git a/collectors/ebpf.plugin/ebpf_apps.h b/collectors/ebpf.plugin/ebpf_apps.h index 338433d404..4b7c92ddb5 100644 --- a/collectors/ebpf.plugin/ebpf_apps.h +++ b/collectors/ebpf.plugin/ebpf_apps.h @@ -20,6 +20,7 @@ #include "ebpf_process.h" #include "ebpf_dcstat.h" +#include "ebpf_disk.h" #include "ebpf_filesystem.h" #include "ebpf_cachestat.h" #include "ebpf_sync.h" diff --git a/collectors/ebpf.plugin/ebpf_disk.c b/collectors/ebpf.plugin/ebpf_disk.c new file mode 100644 index 0000000000..5f62aeaf7c --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_disk.c @@ -0,0 +1,837 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include + +#include "ebpf.h" +#include "ebpf_disk.h" + +struct config disk_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + +static ebpf_local_maps_t disk_maps[] = {{.name = "tbl_disk_iocall", .internal_input = NETDATA_DISK_HISTOGRAM_LENGTH, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; +static ebpf_data_t disk_data; + +static avl_tree_lock disk_tree; +netdata_ebpf_disks_t *disk_list = NULL; + +char *tracepoint_block_type = { "block"} ; +char *tracepoint_block_issue = { "block_rq_issue" }; +char *tracepoint_block_rq_complete = { "block_rq_complete" }; + +static struct bpf_link **probe_links = NULL; +static struct bpf_object *objects = NULL; + +static int was_block_issue_enabled = 0; +static int was_block_rq_complete_enabled = 0; + +static char **dimensions = NULL; +static netdata_syscall_stat_t disk_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS]; +static netdata_publish_syscall_t disk_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS]; + +static int read_thread_closed = 1; + +static netdata_idx_t *disk_hash_values = NULL; +static struct netdata_static_thread disk_threads = {"DISK KERNEL", + NULL, NULL, 1, NULL, + NULL, NULL }; + +ebpf_publish_disk_t *plot_disks = NULL; +pthread_mutex_t plot_mutex; + +/***************************************************************** + * + * FUNCTIONS TO MANIPULATE HARD DISKS + * + *****************************************************************/ + +/** + * Parse start + * + * Parse start address of disk + * + * @param w structure where data is stored + * @param filename variable used to store value + * + * @return It returns 0 on success and -1 otherwise + */ +static inline int ebpf_disk_parse_start(netdata_ebpf_disks_t *w, char *filename) +{ + char content[FILENAME_MAX + 1]; + int fd = open(filename, O_RDONLY, 0); + if (fd < 0) { + return -1; + } + + ssize_t file_length = read(fd, content, 4095); + if (file_length > 0) { + if (file_length > FILENAME_MAX) + file_length = FILENAME_MAX; + + content[file_length] = '\0'; + w->start = strtoul(content, NULL, 10); + } + close(fd); + + return 0; +} + +/** + * Parse uevent + * + * Parse uevent file + * + * @param w structure where data is stored + * @param filename variable used to store value + * + * @return It returns 0 on success and -1 otherwise + */ +static inline int ebpf_parse_uevent(netdata_ebpf_disks_t *w, char *filename) +{ + char content[FILENAME_MAX + 1]; + int fd = open(filename, O_RDONLY, 0); + if (fd < 0) { + return -1; + } + + ssize_t file_length = read(fd, content, FILENAME_MAX); + if (file_length > 0) { + if (file_length > FILENAME_MAX) + file_length = FILENAME_MAX; + + content[file_length] = '\0'; + + char *s = strstr(content, "PARTNAME=EFI"); + if (s) { + w->main->boot_partition = w; + w->flags |= NETDATA_DISK_HAS_EFI; + w->boot_chart = strdupz("disk_bootsector"); + } + } + close(fd); + + return 0; +} + +/** + * Parse Size + * + * @param w structure where data is stored + * @param filename variable used to store value + * + * @return It returns 0 on success and -1 otherwise + */ +static inline int ebpf_parse_size(netdata_ebpf_disks_t *w, char *filename) +{ + char content[FILENAME_MAX + 1]; + int fd = open(filename, O_RDONLY, 0); + if (fd < 0) { + return -1; + } + + ssize_t file_length = read(fd, content, FILENAME_MAX); + if (file_length > 0) { + if (file_length > FILENAME_MAX) + file_length = FILENAME_MAX; + + content[file_length] = '\0'; + w->end = w->start + strtoul(content, NULL, 10) -1; + } + close(fd); + + return 0; +} + +/** + * Read Disk information + * + * Read disk information from /sys/block + * + * @param w structure where data is stored + * @param name disk name + */ +static void ebpf_read_disk_info(netdata_ebpf_disks_t *w, char *name) +{ + static netdata_ebpf_disks_t *main_disk = NULL; + static uint32_t key = 0; + char *path = { "/sys/block" }; + char disk[NETDATA_DISK_NAME_LEN + 1]; + char filename[FILENAME_MAX + 1]; + snprintfz(disk, NETDATA_DISK_NAME_LEN, "%s", name); + size_t length = strlen(disk); + if (!length) { + return; + } + + length--; + size_t curr = length; + while (isdigit((int)disk[length])) { + disk[length--] = '\0'; + } + + // We are looking for partition information, if it is a device we will ignore it. + if (curr == length) { + main_disk = w; + key = MKDEV(w->major, w->minor); + w->bootsector_key = key; + return; + } + w->bootsector_key = key; + w->main = main_disk; + + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/uevent", path, disk, name); + if (ebpf_parse_uevent(w, filename)) + return; + + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/start", path, disk, name); + if (ebpf_disk_parse_start(w, filename)) + return; + + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/size", path, disk, name); + ebpf_parse_size(w, filename); +} + +/** + * New encode dev + * + * New encode algorithm extracted from https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L39 + * + * @param major driver major number + * @param minor driver minor number + * + * @return + */ +static inline uint32_t netdata_new_encode_dev(uint32_t major, uint32_t minor) { + return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); +} + +/** + * Compare disks + * + * Compare major and minor values to add disks to tree. + * + * @param a pointer to netdata_ebpf_disks + * @param b pointer to netdata_ebpf_disks + * + * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b. +*/ +static int ebpf_compare_disks(void *a, void *b) +{ + netdata_ebpf_disks_t *ptr1 = a; + netdata_ebpf_disks_t *ptr2 = b; + + if (ptr1->dev > ptr2->dev) + return 1; + if (ptr1->dev < ptr2->dev) + return -1; + + return 0; +} + +/** + * Update listen table + * + * Update link list when it is necessary. + * + * @param name disk name + * @param major major disk identifier + * @param minor minor disk identifier + * @param current_time current timestamp + */ +static void update_disk_table(char *name, int major, int minor, time_t current_time) +{ + netdata_ebpf_disks_t find; + netdata_ebpf_disks_t *w; + size_t length; + + uint32_t dev = netdata_new_encode_dev(major, minor); + find.dev = dev; + netdata_ebpf_disks_t *ret = (netdata_ebpf_disks_t *) avl_search_lock(&disk_tree, (avl_t *)&find); + if (ret) { // Disk is already present + ret->flags |= NETDATA_DISK_IS_HERE; + ret->last_update = current_time; + return; + } + + netdata_ebpf_disks_t *update_next = disk_list; + if (likely(disk_list)) { + netdata_ebpf_disks_t *move = disk_list; + while (move) { + if (dev == move->dev) + return; + + update_next = move; + move = move->next; + } + + w = callocz(1, sizeof(netdata_ebpf_disks_t)); + length = strlen(name); + if (length >= NETDATA_DISK_NAME_LEN) + length = NETDATA_DISK_NAME_LEN; + + strncpy(w->family, name, length); + w->family[length] = '\0'; + w->major = major; + w->minor = minor; + w->dev = netdata_new_encode_dev(major, minor); + update_next->next = w; + } else { + disk_list = callocz(1, sizeof(netdata_ebpf_disks_t)); + length = strlen(name); + if (length >= NETDATA_DISK_NAME_LEN) + length = NETDATA_DISK_NAME_LEN; + + strncpy(disk_list->family, name, length); + disk_list->family[length] = '\0'; + disk_list->major = major; + disk_list->minor = minor; + disk_list->dev = netdata_new_encode_dev(major, minor); + + w = disk_list; + } + + ebpf_read_disk_info(w, name); + + netdata_ebpf_disks_t *check; + check = (netdata_ebpf_disks_t *) avl_insert_lock(&disk_tree, (avl_t *)w); + if (check != w) + error("Internal error, cannot insert the AVL tree."); + +#ifdef NETDATA_INTERNAL_CHECKS + info("The Latency is monitoring the hard disk %s (Major = %d, Minor = %d, Device = %u)", name, major, minor,w->dev); +#endif + + w->flags |= NETDATA_DISK_IS_HERE; +} + +/** + * Read Local Disks + * + * Parse /proc/partitions to get block disks used to measure latency. + * + * @return It returns 0 on success and -1 otherwise + */ +static int read_local_disks() +{ + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_EBPF_PROC_PARTITIONS); + procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT); + if (!ff) + return -1; + + ff = procfile_readall(ff); + if (!ff) + return -1; + + size_t lines = procfile_lines(ff), l; + time_t current_time = now_realtime_sec(); + for(l = 2; l < lines ;l++) { + size_t words = procfile_linewords(ff, l); + // This is header or end of file + if (unlikely(words < 4)) + continue; + + int major = (int)strtol(procfile_lineword(ff, l, 0), NULL, 10); + // The main goal of this thread is to measure block devices, so any block device with major number + // smaller than 7 according /proc/devices is not "important". + if (major > 7) { + int minor = (int)strtol(procfile_lineword(ff, l, 1), NULL, 10); + update_disk_table(procfile_lineword(ff, l, 3), major, minor, current_time); + } + } + + procfile_close(ff); + + return 0; +} + +/** + * Update disks + * + * @param em main thread structure + */ +void ebpf_update_disks(ebpf_module_t *em) +{ + static time_t update_time = 0; + time_t curr = now_realtime_sec(); + if (curr < update_time) + return; + + update_time = curr + 5 * em->update_time; + + (void)read_local_disks(); +} + +/***************************************************************** + * + * FUNCTIONS TO CLOSE THE THREAD + * + *****************************************************************/ + +/** + * Disk disable tracepoints + * + * Disable tracepoints when the plugin was responsible to enable it. + */ +static void ebpf_disk_disable_tracepoints() +{ + char *default_message = { "Cannot disable the tracepoint" }; + if (!was_block_issue_enabled) { + if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_issue)) + error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_issue); + } + + if (!was_block_rq_complete_enabled) { + if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete)) + error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_rq_complete); + } +} + +/** + * Cleanup plot disks + * + * Clean disk list + */ +static void ebpf_cleanup_plot_disks() +{ + ebpf_publish_disk_t *move = plot_disks, *next; + while (move) { + next = move->next; + + freez(move); + + move = next; + } +} + +/** + * Cleanup Disk List + */ +static void ebpf_cleanup_disk_list() +{ + netdata_ebpf_disks_t *move = disk_list; + while (move) { + netdata_ebpf_disks_t *next = move->next; + + freez(move->histogram.name); + freez(move->boot_chart); + freez(move); + + move = next; + } +} + +/** + * Clean up the main thread. + * + * @param ptr thread data. + */ +static void ebpf_disk_cleanup(void *ptr) +{ + ebpf_disk_disable_tracepoints(); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + if (!em->enabled) + return; + + heartbeat_t hb; + heartbeat_init(&hb); + uint32_t tick = 2 * USEC_PER_MS; + while (!read_thread_closed) { + usec_t dt = heartbeat_next(&hb, tick); + UNUSED(dt); + } + + if (dimensions) + ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS); + + freez(disk_hash_values); + freez(disk_threads.thread); + pthread_mutex_destroy(&plot_mutex); + + ebpf_cleanup_plot_disks(); + ebpf_cleanup_disk_list(); + + if (probe_links) { + struct bpf_program *prog; + size_t i = 0 ; + bpf_object__for_each_program(prog, objects) { + bpf_link__destroy(probe_links[i]); + i++; + } + bpf_object__close(objects); + } +} + +/***************************************************************** + * + * MAIN LOOP + * + *****************************************************************/ + +/** + * Fill Plot list + * + * @param ptr a pointer for current disk + */ +static void ebpf_fill_plot_disks(netdata_ebpf_disks_t *ptr) +{ + pthread_mutex_lock(&plot_mutex); + ebpf_publish_disk_t *w; + if (likely(plot_disks)) { + ebpf_publish_disk_t *move = plot_disks, *store = plot_disks; + while (move) { + if (move->plot == ptr) { + pthread_mutex_unlock(&plot_mutex); + return; + } + + store = move; + move = move->next; + } + + w = callocz(1, sizeof(ebpf_publish_disk_t)); + w->plot = ptr; + store->next = w; + } else { + plot_disks = callocz(1, sizeof(ebpf_publish_disk_t)); + plot_disks->plot = ptr; + } + pthread_mutex_unlock(&plot_mutex); + + ptr->flags |= NETDATA_DISK_ADDED_TO_PLOT_LIST; +} + +/** + * Read hard disk table + * + * @param table file descriptor for table + * + * Read the table with number of calls for all functions + */ +static void read_hard_disk_tables(int table) +{ + netdata_idx_t *values = disk_hash_values; + block_key_t key = {}; + block_key_t next_key = {}; + + netdata_ebpf_disks_t *ret = NULL; + + while (bpf_map_get_next_key(table, &key, &next_key) == 0) { + int test = bpf_map_lookup_elem(table, &key, values); + if (test < 0) { + key = next_key; + continue; + } + + netdata_ebpf_disks_t find; + find.dev = key.dev; + + if (likely(ret)) { + if (find.dev != ret->dev) + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); + } else + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); + + // Disk was inserted after we parse /proc/partitions + if (!ret) { + if (read_local_disks()) { + key = next_key; + continue; + } + + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find); + if (!ret) { + // We should never reach this point, but we are adding it to keep a safe code + key = next_key; + continue; + } + } + + uint64_t total = 0; + int i; + int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs; + for (i = 0; i < end; i++) { + total += values[i]; + } + + ret->histogram.histogram[key.bin] = total; + + if (!(ret->flags & NETDATA_DISK_ADDED_TO_PLOT_LIST)) + ebpf_fill_plot_disks(ret); + + key = next_key; + } +} + +/** + * Disk read hash + * + * This is the thread callback. + * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket. + * + * @param ptr It is a NULL value for this thread. + * + * @return It always returns NULL. + */ +void *ebpf_disk_read_hash(void *ptr) +{ + heartbeat_t hb; + heartbeat_init(&hb); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + + usec_t step = NETDATA_LATENCY_DISK_SLEEP_MS * em->update_time; + while (!close_ebpf_plugin) { + usec_t dt = heartbeat_next(&hb, step); + (void)dt; + + read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd); + } + + return NULL; +} + +/** + * Obsolete Hard Disk charts + * + * @param w the structure with necessary information to create the chart + * + * Make Hard disk charts and fill chart name + */ +static void ebpf_obsolete_hd_charts(netdata_ebpf_disks_t *w) +{ + ebpf_write_chart_obsolete(w->histogram.name, w->family, w->histogram.title, EBPF_COMMON_DIMENSION_CALL, + w->family, "disk.latency_io", NETDATA_EBPF_CHART_TYPE_STACKED, + w->histogram.order); + + w->flags = 0; +} + +/** + * Create Hard Disk charts + * + * @param w the structure with necessary information to create the chart + * + * Make Hard disk charts and fill chart name + */ +static void ebpf_create_hd_charts(netdata_ebpf_disks_t *w) +{ + int order = NETDATA_CHART_PRIO_DISK_LATENCY; + char *family = w->family; + + w->histogram.name = strdupz("disk_latency_io"); + w->histogram.title = NULL; + w->histogram.order = order; + + ebpf_create_chart(w->histogram.name, family, "Disk latency", EBPF_COMMON_DIMENSION_CALL, + family, "disk.latency_io", NETDATA_EBPF_CHART_TYPE_STACKED, order, + ebpf_create_global_dimension, disk_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS); + order++; + + w->flags |= NETDATA_DISK_CHART_CREATED; +} + +/** + * Remove pointer from plot + * + * Remove pointer from plot list when the disk is not present. + */ +static void ebpf_remove_pointer_from_plot_disk(ebpf_module_t *em) +{ + time_t current_time = now_realtime_sec(); + time_t limit = 10 * em->update_time; + pthread_mutex_lock(&plot_mutex); + ebpf_publish_disk_t *move = plot_disks, *prev = plot_disks; + while (move) { + netdata_ebpf_disks_t *ned = move->plot; + uint32_t flags = ned->flags; + + if (!(flags & NETDATA_DISK_IS_HERE) && ((current_time - ned->last_update) > limit)) { + ebpf_obsolete_hd_charts(ned); + if (move == plot_disks) { + freez(move); + plot_disks = NULL; + break; + } else { + prev->next = move->next; + ebpf_publish_disk_t *clean = move; + move = move->next; + freez(clean); + continue; + } + } + + prev = move; + move = move->next; + } + pthread_mutex_unlock(&plot_mutex); +} + +/** + * Send Hard disk data + * + * Send hard disk information to Netdata. + */ +static void ebpf_latency_send_hd_data() +{ + pthread_mutex_lock(&plot_mutex); + if (!plot_disks) { + pthread_mutex_unlock(&plot_mutex); + return; + } + + ebpf_publish_disk_t *move = plot_disks; + while (move) { + netdata_ebpf_disks_t *ned = move->plot; + uint32_t flags = ned->flags; + if (!(flags & NETDATA_DISK_CHART_CREATED)) { + ebpf_create_hd_charts(ned); + } + + if ((flags & NETDATA_DISK_CHART_CREATED)) { + write_histogram_chart(ned->histogram.name, ned->family, + ned->histogram.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); + } + + ned->flags &= ~NETDATA_DISK_IS_HERE; + + move = move->next; + } + pthread_mutex_unlock(&plot_mutex); +} + +/** +* Main loop for this collector. +*/ +static void disk_collector(ebpf_module_t *em) +{ + disk_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); + disk_threads.thread = mallocz(sizeof(netdata_thread_t)); + disk_threads.start_routine = ebpf_disk_read_hash; + + netdata_thread_create(disk_threads.thread, disk_threads.name, NETDATA_THREAD_OPTION_JOINABLE, + ebpf_disk_read_hash, em); + + + read_thread_closed = 0; + while (!close_ebpf_plugin) { + pthread_mutex_lock(&collect_data_mutex); + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex); + + pthread_mutex_lock(&lock); + ebpf_remove_pointer_from_plot_disk(em); + ebpf_latency_send_hd_data(); + + pthread_mutex_unlock(&lock); + pthread_mutex_unlock(&collect_data_mutex); + + ebpf_update_disks(em); + } + read_thread_closed = 1; +} + +/***************************************************************** + * + * EBPF DISK THREAD + * + *****************************************************************/ + +/** + * Enable tracepoints + * + * Enable necessary tracepoints for thread. + * + * @return It returns 0 on success and -1 otherwise + */ +static int ebpf_disk_enable_tracepoints() +{ + int test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_issue); + if (test == -1) + return -1; + else if (!test) { + if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_issue)) + return -1; + } + was_block_issue_enabled = test; + + test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_rq_complete); + if (test == -1) + return -1; + else if (!test) { + if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete)) + return -1; + } + was_block_rq_complete_enabled = test; + + return 0; +} + +/** + * Disk thread + * + * Thread used to generate disk charts. + * + * @param ptr a pointer to `struct ebpf_module` + * + * @return It always return NULL + */ +void *ebpf_disk_thread(void *ptr) +{ + netdata_thread_cleanup_push(ebpf_disk_cleanup, ptr); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + em->maps = disk_maps; + + fill_ebpf_data(&disk_data); + + if (!em->enabled) + goto enddisk; + + if (ebpf_update_kernel(&disk_data)) { + goto enddisk; + } + + if (ebpf_disk_enable_tracepoints()) { + em->enabled = CONFIG_BOOLEAN_NO; + goto enddisk; + } + + avl_init_lock(&disk_tree, ebpf_compare_disks); + if (read_local_disks()) { + em->enabled = CONFIG_BOOLEAN_NO; + goto enddisk; + } + + if (pthread_mutex_init(&plot_mutex, NULL)) { + error("Cannot initialize local mutex"); + goto enddisk; + } + + probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, disk_data.map_fd); + if (!probe_links) { + goto enddisk; + } + + int algorithms[NETDATA_EBPF_HIST_MAX_BINS]; + ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX); + dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS); + + ebpf_global_labels(disk_aggregated_data, disk_publish_aggregated, dimensions, dimensions, algorithms, + NETDATA_EBPF_HIST_MAX_BINS); + + disk_collector(em); + +enddisk: + netdata_thread_cleanup_pop(1); + + return NULL; +} diff --git a/collectors/ebpf.plugin/ebpf_disk.h b/collectors/ebpf.plugin/ebpf_disk.h new file mode 100644 index 0000000000..8727aabac2 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_disk.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EBPF_DISK_H +#define NETDATA_EBPF_DISK_H 1 + +#include "libnetdata/avl/avl.h" +#include "libnetdata/ebpf/ebpf.h" + +#define NETDATA_EBPF_PROC_PARTITIONS "/proc/partitions" + +#define NETDATA_LATENCY_DISK_SLEEP_MS 650000ULL + +// Decode function extracted from: https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L7 +#define MINORBITS 20 +#define MKDEV(ma,mi) (((ma) << MINORBITS) | (mi)) + +enum netdata_latency_disks_flags { + NETDATA_DISK_ADDED_TO_PLOT_LIST = 1, + NETDATA_DISK_CHART_CREATED = 2, + NETDATA_DISK_IS_HERE = 4, + NETDATA_DISK_HAS_EFI = 8 +}; + +/* + * The definition (DISK_NAME_LEN) has been a stable value since Kernel 3.0, + * I decided to bring it as internal definition, to avoid include linux/genhd.h. + */ +#define NETDATA_DISK_NAME_LEN 32 +typedef struct netdata_ebpf_disks { + // Search + avl_t avl; + uint32_t dev; + uint32_t major; + uint32_t minor; + uint32_t bootsector_key; + uint64_t start; // start sector + uint64_t end; // end sector + + // Print information + char family[NETDATA_DISK_NAME_LEN + 1]; + char *boot_chart; + + netdata_ebpf_histogram_t histogram; + + uint32_t flags; + time_t last_update; + + struct netdata_ebpf_disks *main; + struct netdata_ebpf_disks *boot_partition; + struct netdata_ebpf_disks *next; +} netdata_ebpf_disks_t; + +enum ebpf_disk_tables { + NETDATA_DISK_READ +}; + +typedef struct block_key { + uint32_t bin; + uint32_t dev; +} block_key_t; + +typedef struct netdata_ebpf_publish_disk { + netdata_ebpf_disks_t *plot; + struct netdata_ebpf_publish_disk *next; +} ebpf_publish_disk_t; + +extern struct config disk_config; + +extern void *ebpf_disk_thread(void *ptr); + +#endif /* NETDATA_EBPF_DISK_H */ + diff --git a/collectors/ebpf.plugin/ebpf_filesystem.c b/collectors/ebpf.plugin/ebpf_filesystem.c index 92ef08f660..7d28c96a72 100644 --- a/collectors/ebpf.plugin/ebpf_filesystem.c +++ b/collectors/ebpf.plugin/ebpf_filesystem.c @@ -36,8 +36,8 @@ struct netdata_static_thread filesystem_threads = {"EBPF FS READ", NULL, NULL }; static int read_thread_closed = 1; -static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_FILESYSTEM_MAX_BINS]; -static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_FILESYSTEM_MAX_BINS]; +static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS]; +static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS]; char **dimensions = NULL; static netdata_idx_t *filesystem_hash_values = NULL; @@ -114,7 +114,7 @@ static void ebpf_create_fs_charts() title, EBPF_COMMON_DIMENSION_CALL, family, NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension, - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS); + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS); order++; snprintfz(title, 255, "%s latency for each write request.", efp->filesystem); @@ -126,7 +126,7 @@ static void ebpf_create_fs_charts() title, EBPF_COMMON_DIMENSION_CALL, family, NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension, - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS); + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS); order++; snprintfz(title, 255, "%s latency for each open request.", efp->filesystem); @@ -138,7 +138,7 @@ static void ebpf_create_fs_charts() title, EBPF_COMMON_DIMENSION_CALL, family, NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension, - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS); + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS); order++; snprintfz(title, 255, "%s latency for each sync request.", efp->filesystem); @@ -150,7 +150,7 @@ static void ebpf_create_fs_charts() title, EBPF_COMMON_DIMENSION_CALL, family, NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension, - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS); + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS); order++; efp->flags |= NETDATA_FILESYSTEM_FLAG_CHART_CREATED; } @@ -197,10 +197,10 @@ int ebpf_filesystem_initialize_ebpf_data(ebpf_module_t *em) em->thread_name = saved_name; if (!dimensions) { - dimensions = ebpf_fill_histogram_dimension(NETDATA_FILESYSTEM_MAX_BINS); + dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS); - memset(filesystem_aggregated_data, 0 , NETDATA_FILESYSTEM_MAX_BINS * sizeof(netdata_syscall_stat_t)); - memset(filesystem_publish_aggregated, 0 , NETDATA_FILESYSTEM_MAX_BINS * sizeof(netdata_publish_syscall_t)); + memset(filesystem_aggregated_data, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_syscall_stat_t)); + memset(filesystem_publish_aggregated, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_publish_syscall_t)); filesystem_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); } @@ -351,7 +351,8 @@ static void ebpf_filesystem_cleanup(void *ptr) ebpf_cleanup_publish_syscall(filesystem_publish_aggregated); ebpf_filesystem_cleanup_ebpf_data(); - ebpf_histogram_dimension_cleanup(dimensions, NETDATA_FILESYSTEM_MAX_BINS); + if (dimensions) + ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS); freez(filesystem_hash_values); } @@ -421,8 +422,8 @@ static void read_filesystem_table(ebpf_filesystem_partitions_t *efp) total += values[i]; } - if (idx >= NETDATA_FILESYSTEM_MAX_BINS) - idx = NETDATA_FILESYSTEM_MAX_BINS - 1; + if (idx >= NETDATA_EBPF_HIST_MAX_BINS) + idx = NETDATA_EBPF_HIST_MAX_BINS - 1; w->histogram[idx] = total; } } @@ -481,30 +482,6 @@ void *ebpf_filesystem_read_hash(void *ptr) return NULL; } -/** - * Call the necessary functions to create a name. - * - * @param family family name - * @param name chart name - * @param hist0 histogram values - * @param end number of bins that will be sent to Netdata. - * - * @return It returns a variable tha maps the charts that did not have zero values. - */ -static void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, uint32_t end) -{ - write_begin_chart(family, name); - - uint32_t i; - for (i = 0; i < end; i++) { - write_chart_dimension(dimensions[i], (long long) hist[i]); - } - - write_end_chart(); - - fflush(stdout); -} - /** * Send Hard disk data * @@ -518,16 +495,16 @@ static void ebpf_histogram_send_data() ebpf_filesystem_partitions_t *efp = &localfs[i]; if ((efp->flags & test) == NETDATA_FILESYSTEM_FLAG_HAS_PARTITION) { write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hread.name, - efp->hread.histogram, NETDATA_FILESYSTEM_MAX_BINS); + efp->hread.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hwrite.name, - efp->hwrite.histogram, NETDATA_FILESYSTEM_MAX_BINS); + efp->hwrite.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name, - efp->hopen.histogram, NETDATA_FILESYSTEM_MAX_BINS); + efp->hopen.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hsync.name, - efp->hsync.histogram, NETDATA_FILESYSTEM_MAX_BINS); + efp->hsync.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS); } } } @@ -612,10 +589,10 @@ void *ebpf_filesystem_thread(void *ptr) goto endfilesystem; } - int algorithms[NETDATA_FILESYSTEM_MAX_BINS]; - ebpf_fill_algorithms(algorithms, NETDATA_FILESYSTEM_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX); + int algorithms[NETDATA_EBPF_HIST_MAX_BINS]; + ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX); ebpf_global_labels(filesystem_aggregated_data, filesystem_publish_aggregated, dimensions, dimensions, - algorithms, NETDATA_FILESYSTEM_MAX_BINS); + algorithms, NETDATA_EBPF_HIST_MAX_BINS); pthread_mutex_lock(&lock); ebpf_create_fs_charts(); diff --git a/collectors/ebpf.plugin/ebpf_filesystem.h b/collectors/ebpf.plugin/ebpf_filesystem.h index b9f06136e1..c7f621fd07 100644 --- a/collectors/ebpf.plugin/ebpf_filesystem.h +++ b/collectors/ebpf.plugin/ebpf_filesystem.h @@ -5,7 +5,6 @@ #include "ebpf.h" -#define NETDATA_FILESYSTEM_MAX_BINS 24UL #define NETDATA_FS_MAX_DIST_NAME 64UL #define NETDATA_FILESYSTEM_CONFIG_NAME "filesystem" @@ -32,14 +31,6 @@ enum netdata_filesystem_flags { NETDATA_FILESYSTEM_REMOVE_CHARTS = 16 }; -typedef struct netdata_ebpf_histogram { - char *name; - char *title; - int order; - uint64_t histogram[NETDATA_FILESYSTEM_MAX_BINS]; -} netdata_ebpf_histogram_t; - - enum netdata_filesystem_table { NETDATA_MAIN_FS_TABLE, NETDATA_ADDR_FS_TABLE diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c index fa959b8dfd..dc2b2cfa8a 100644 --- a/libnetdata/ebpf/ebpf.c +++ b/libnetdata/ebpf/ebpf.c @@ -701,3 +701,114 @@ void ebpf_histogram_dimension_cleanup(char **ptr, size_t length) } freez(ptr); } + +//---------------------------------------------------------------------------------------------------------------------- + +/** + * Open tracepoint path + * + * @param filename pointer to store the path + * @param length file length + * @param subsys is the name of your subsystem. + * @param eventname is the name of the event to trace. + * @param flags flags used with syscall open + * + * @return it returns a positive value on success and a negative otherwise. + */ +static inline int ebpf_open_tracepoint_path(char *filename, size_t length, char *subsys, char *eventname, int flags) +{ + snprintfz(filename, length, "%s/events/%s/%s/enable", NETDATA_DEBUGFS, subsys, eventname); + return open(filename, flags, 0); +} + +/** + * Is tracepoint enabled + * + * Check whether the tracepoint is enabled. + * + * @param subsys is the name of your subsystem. + * @param eventname is the name of the event to trace. + * + * @return it returns 1 when it is enabled, 0 when it is disabled and -1 on error. + */ +int ebpf_is_tracepoint_enabled(char *subsys, char *eventname) +{ + char text[FILENAME_MAX + 1]; + int fd = ebpf_open_tracepoint_path(text, FILENAME_MAX, subsys, eventname, O_RDONLY); + if (fd < 0) { + return -1; + } + + ssize_t length = read(fd, text, 1); + if (length != 1) { + close(fd); + return -1; + } + close(fd); + + return (text[0] == '1') ? CONFIG_BOOLEAN_YES : CONFIG_BOOLEAN_NO; +} + +/** + * Change Tracing values + * + * Change value for specific tracepoint enabling or disabling it according value given. + * + * @param subsys is the name of your subsystem. + * @param eventname is the name of the event to trace. + * @param value a value to enable (1) or disable (0) a tracepoint. + * + * @return It returns 0 on success and -1 otherwise + */ +static int ebpf_change_tracing_values(char *subsys, char *eventname, char *value) +{ + if (strcmp("0", value) && strcmp("1", value)) { + error("Invalid value given to either enable or disable a tracepoint."); + return -1; + } + + char filename[1024]; + int fd = ebpf_open_tracepoint_path(filename, 1023, subsys, eventname, O_WRONLY); + if (fd < 0) { + return -1; + } + + ssize_t written = write(fd, value, strlen(value)); + if (written < 0) { + close(fd); + return -1; + } + + close(fd); + return 0; +} + +/** + * Enable tracing values + * + * Enable a tracepoint on a system + * + * @param subsys is the name of your subsystem. + * @param eventname is the name of the event to trace. + * + * @return It returns 0 on success and -1 otherwise + */ +int ebpf_enable_tracing_values(char *subsys, char *eventname) +{ + return ebpf_change_tracing_values(subsys, eventname, "1"); +} + +/** + * Disable tracing values + * + * Disable tracing points enabled by collector + * + * @param subsys is the name of your subsystem. + * @param eventname is the name of the event to trace. + * + * @return It returns 0 on success and -1 otherwise + */ +int ebpf_disable_tracing_values(char *subsys, char *eventname) +{ + return ebpf_change_tracing_values(subsys, eventname, "0"); +} diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h index 9896b1d616..4d856372cb 100644 --- a/libnetdata/ebpf/ebpf.h +++ b/libnetdata/ebpf/ebpf.h @@ -184,6 +184,25 @@ extern void ebpf_update_names(ebpf_specify_name_t *opt, ebpf_module_t *em); extern void ebpf_load_addresses(ebpf_addresses_t *fa, int fd); extern void ebpf_fill_algorithms(int *algorithms, size_t length, int algorithm); extern char **ebpf_fill_histogram_dimension(size_t maximum); + +// Histogram +#define NETDATA_EBPF_HIST_MAX_BINS 24UL +#define NETDATA_DISK_MAX 256U +#define NETDATA_DISK_HISTOGRAM_LENGTH (NETDATA_DISK_MAX * NETDATA_EBPF_HIST_MAX_BINS) + +typedef struct netdata_ebpf_histogram { + char *name; + char *title; + int order; + uint64_t histogram[NETDATA_EBPF_HIST_MAX_BINS]; +} netdata_ebpf_histogram_t; + extern void ebpf_histogram_dimension_cleanup(char **ptr, size_t length); +// Tracepoint helpers +// For more information related to tracepoints read https://www.kernel.org/doc/html/latest/trace/tracepoints.html +extern int ebpf_is_tracepoint_enabled(char *subsys, char *eventname); +extern int ebpf_enable_tracing_values(char *subsys, char *eventname); +extern int ebpf_disable_tracing_values(char *subsys, char *eventname); + #endif /* NETDATA_EBPF_H */ diff --git a/packaging/ebpf.checksums b/packaging/ebpf.checksums index 255750aada..0d326919f7 100644 --- a/packaging/ebpf.checksums +++ b/packaging/ebpf.checksums @@ -1,3 +1,3 @@ -a2849a54f43f9419419298a0e06de7da52b1ecb74e3c5e788540f7fdd380862a netdata-kernel-collector-glibc-v0.7.1.1.tar.xz -15fb92966fef3fe8b6f6f08f15c57270b5b37071b52834073097076c4897c0e1 netdata-kernel-collector-musl-v0.7.1.1.tar.xz -bdfa64cfc9a9d457252dd824435ac5e7f107c9a30fc6d915b294ae9fb1cd5205 netdata-kernel-collector-static-v0.7.1.1.tar.xz +1e2441471f2f399776e1b61fa0e55df6937e4e120a003bda73b510a2b99df583 netdata-kernel-collector-glibc-v0.7.2.4.tar.xz +76f7c342e2eb50d306b6a12d42be32f42ae74fe36ea48d0b5021973aae9be666 netdata-kernel-collector-musl-v0.7.2.4.tar.xz +8a57cec811512c0fc9f8c06bcf2e4c01553d7fdd0a8123c578fd0372f5ea768f netdata-kernel-collector-static-v0.7.2.4.tar.xz diff --git a/packaging/ebpf.version b/packaging/ebpf.version index c4e0bdea64..7d5069569d 100644 --- a/packaging/ebpf.version +++ b/packaging/ebpf.version @@ -1 +1 @@ -v0.7.1.1 +v0.7.2.4 diff --git a/web/gui/dashboard_info.js b/web/gui/dashboard_info.js index a1188e332d..32b2923df5 100644 --- a/web/gui/dashboard_info.js +++ b/web/gui/dashboard_info.js @@ -1507,6 +1507,10 @@ netdataDashboard.context = { height: 0.5, info: 'The average service time for completed I/O operations. This metric is calculated using the total busy time of the disk and the number of completed operations. If the disk is able to execute multiple parallel operations the reporting average service time will be misleading.' }, + 'disk.latency_io': { + height: 0.5, + info: 'Disk I/O latency is the time it takes for an I/O request to be completed. Latency is the single most important metric to focus on when it comes to storage performance, under most circumstances. For hard drives, an average latency somewhere between 10 to 20 ms can be considered acceptable. For SSD (Solid State Drives), depending on the workload it should never reach higher than 1-3 ms. In most cases, workloads will experience less than 1ms latency numbers. The dimensions refer to time intervals. This chart is based on the bio_tracepoints tool of the ebpf_exporter.' + }, 'disk.avgsz': { height: 0.5, info: 'The average I/O operation size.' -- cgit v1.2.3