From 18ec989b21eb1c6cf1f736e27b6b89b1a3108b34 Mon Sep 17 00:00:00 2001 From: thiagoftsm Date: Wed, 13 May 2020 16:56:42 +0000 Subject: Rename eBPF collector (#8822) We renamed eBPF collector for a more meaningful name. --- .gitignore | 4 +- CMakeLists.txt | 14 +- Makefile.am | 12 +- README.md | 2 + collectors/Makefile.am | 2 +- collectors/ebpf.plugin/Makefile.am | 13 + collectors/ebpf.plugin/README.md | 219 ++++ collectors/ebpf.plugin/ebpf.c | 1152 ++++++++++++++++++++++ collectors/ebpf.plugin/ebpf.conf | 2 + collectors/ebpf.plugin/ebpf.h | 155 +++ collectors/ebpf_process.plugin/Makefile.am | 12 - collectors/ebpf_process.plugin/README.md | 220 ----- collectors/ebpf_process.plugin/ebpf_process.c | 1152 ---------------------- collectors/ebpf_process.plugin/ebpf_process.conf | 2 - collectors/ebpf_process.plugin/ebpf_process.h | 155 --- collectors/plugins.d/plugins_d.c | 2 +- configure.ac | 14 +- docs/generator/buildyaml.sh | 2 +- netdata-installer.sh | 25 +- netdata.spec.in | 4 +- 20 files changed, 1592 insertions(+), 1571 deletions(-) create mode 100644 collectors/ebpf.plugin/Makefile.am create mode 100644 collectors/ebpf.plugin/README.md create mode 100644 collectors/ebpf.plugin/ebpf.c create mode 100644 collectors/ebpf.plugin/ebpf.conf create mode 100644 collectors/ebpf.plugin/ebpf.h delete mode 100644 collectors/ebpf_process.plugin/Makefile.am delete mode 100644 collectors/ebpf_process.plugin/README.md delete mode 100644 collectors/ebpf_process.plugin/ebpf_process.c delete mode 100644 collectors/ebpf_process.plugin/ebpf_process.conf delete mode 100644 collectors/ebpf_process.plugin/ebpf_process.h diff --git a/.gitignore b/.gitignore index b9991bff00..db056ef1be 100644 --- a/.gitignore +++ b/.gitignore @@ -66,8 +66,8 @@ slabinfo.plugin cgroup-network !cgroup-network/ -ebpf_process.plugin -!ebpf_process.plugin/ +ebpf.plugin +!ebpf.plugin/ # protoc generated files *.pb.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index ca0ab99946..e14e99ecbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -408,7 +408,7 @@ set(SLABINFO_PLUGIN_FILES ) set(EBPF_PROCESS_PLUGIN_FILES - collectors/ebpf_process.plugin/ebpf_process.c + collectors/ebpf.plugin/ebpf.c ) set(PROC_PLUGIN_FILES @@ -974,13 +974,13 @@ ENDIF() # ebpf_process.plugin IF(ENABLE_PLUGIN_EBPF) - message(STATUS "ebpf_process.plugin: enabled") - add_executable(ebpf_process.plugin config.h ${EBPF_PROCESS_PLUGIN_FILES}) - target_link_libraries (ebpf_process.plugin libnetdata ${NETDATA_COMMON_LIBRARIES}) - target_include_directories(ebpf_process.plugin PUBLIC ${NETDATA_COMMON_INCLUDE_DIRS}) - target_compile_options(ebpf_process.plugin PUBLIC ${NETDATA_COMMON_CFLAGS}) + message(STATUS "ebpf.plugin: enabled") + add_executable(ebpf.plugin config.h ${EBPF_PROCESS_PLUGIN_FILES}) + target_link_libraries (ebpf.plugin libnetdata ${NETDATA_COMMON_LIBRARIES}) + target_include_directories(ebpf.plugin PUBLIC ${NETDATA_COMMON_INCLUDE_DIRS}) + target_compile_options(ebpf.plugin PUBLIC ${NETDATA_COMMON_CFLAGS}) ELSE() - message(STATUS "ebpf_process.plugin: disabled") + message(STATUS "ebpf.plugin: disabled") ENDIF() diff --git a/Makefile.am b/Makefile.am index e39875eae3..298d4550bf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -265,9 +265,9 @@ PERF_PLUGIN_FILES = \ $(LIBNETDATA_FILES) \ $(NULL) -EBPF_PROCESS_PLUGIN_FILES = \ - collectors/ebpf_process.plugin/ebpf_process.c \ - collectors/ebpf_process.plugin/ebpf_process.h \ +EBPF_PLUGIN_FILES = \ + collectors/ebpf.plugin/ebpf.c \ + collectors/ebpf.plugin/ebpf.h \ $(LIBNETDATA_FILES) \ $(NULL) @@ -697,9 +697,9 @@ if ENABLE_PLUGIN_FREEIPMI endif if ENABLE_PLUGIN_EBPF - plugins_PROGRAMS += ebpf_process.plugin - ebpf_process_plugin_SOURCES = $(EBPF_PROCESS_PLUGIN_FILES) - ebpf_process_plugin_LDADD = \ + plugins_PROGRAMS += ebpf.plugin + ebpf_plugin_SOURCES = $(EBPF_PLUGIN_FILES) + ebpf_plugin_LDADD = \ $(NETDATA_COMMON_LIBS) \ $(NULL) endif diff --git a/README.md b/README.md index 90b8cf3ed1..63c6c4c972 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,8 @@ Our new documentation experience is now available at **[Netdata Learn](https://l While Learn only features documentation for now, we plan on releasing more types of educational content serving the Agent's open-source community of developers, sysadmins, and DevOps folks. We'll have more to announce soon, but in the meantime, we hope you enjoy what we believe is a smoother (and prettier) docs experience. +As part of the ongoing work to polish our **eBPF collector tech preview**, we've now proven the collector's performance is very good, and have vastly expanded the number of operating system versions the collector works on. Learn how to [enable it](https://docs.netdata.cloud/collectors/ebpf.plugin/) in our documentation. We've also extensively stress-tested the eBPF collector and found that it's impressively fast given the depth of metrics it collects! Read up on our benchmarking analysis [on GitHub](https://github.com/netdata/netdata/issues/8195). + --- See more news and previous releases at our [blog](https://blog.netdata.cloud) or our [releases diff --git a/collectors/Makefile.am b/collectors/Makefile.am index 9bb5295813..460612c685 100644 --- a/collectors/Makefile.am +++ b/collectors/Makefile.am @@ -24,7 +24,7 @@ SUBDIRS = \ python.d.plugin \ slabinfo.plugin \ statsd.plugin \ - ebpf_process.plugin \ + ebpf.plugin \ tc.plugin \ $(NULL) diff --git a/collectors/ebpf.plugin/Makefile.am b/collectors/ebpf.plugin/Makefile.am new file mode 100644 index 0000000000..719c236367 --- /dev/null +++ b/collectors/ebpf.plugin/Makefile.am @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) + +dist_libconfig_DATA = \ + ebpf.conf \ + $(NULL) + diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md new file mode 100644 index 0000000000..bf00cac167 --- /dev/null +++ b/collectors/ebpf.plugin/README.md @@ -0,0 +1,219 @@ + + +# eBPF monitoring with Netdata + +This collector plugin uses eBPF (Extended Berkeley Packet Filter) to monitor system calls inside your operating system's +kernel. For now, the main goal of this plugin is to monitor IO and process management on the host where it is running. + +
+ An example of VFS charts, made possible by the eBPF collector plugin +
An example of VFS charts, made possible by the eBPF collector plugin
+
+ +With this eBPF collector, you can monitor sophisticated system-level metrics about your complex applications while +maintaining Netdata's [high standards for performance](#performance). + +## Enable the collector on Linux + +eBPF is only available on Linux systems, which means this collector only works on Linux. + +The collector is currently in an _alpha_ stage, as we are still working on improving compatibility with more Linux +distributions and versions, and to ensure the collector works as expected. + +Follow the next few steps to ensure compatibility, prepare your system, install Netdata with eBPF compiled, and enable +the collector. + +### Ensure kernel compatibility + +To enable this plugin and its collector, you must be on a Linux system with a kernel that is more recent than `4.11.0` +and compiled with the option `CONFIG_KPROBES=y`. You can verify whether your kernel has this option enabled by running +the following commands: + +```bash +grep CONFIG_KPROBES=y /boot/config-$(uname -r) +zgrep CONFIG_KPROBES=y /proc/config.gz +``` + +If `Kprobes` is enabled, you will see `CONFIG_KPROBES=y` as the command's output, and can skip ahead to the next step: [mount `debugfs` and `tracefs`](#mount-debugfs-and-tracefs). + +If you don't see `CONFIG_KPROBES=y` for any of the commands above, you will have to recompile your kernel to enable it. + +The process of recompiling Linux kernels varies based on your distribution and version. Read the documentation for your +system's distribution to learn more about the specific workflow for recompiling the kernel, ensuring that you set the +`CONFIG_KPROBES` setting to `y` in the process. + +- [Ubuntu](https://wiki.ubuntu.com/Kernel/BuildYourOwnKernel) +- [Debian](https://kernel-team.pages.debian.net/kernel-handbook/ch-common-tasks.html#s-common-official) +- [Fedora](https://fedoraproject.org/wiki/Building_a_custom_kernel) +- [CentOS](https://wiki.centos.org/HowTos/Custom_Kernel) +- [Arch Linux](https://wiki.archlinux.org/index.php/Kernel/Traditional_compilation) +- [Slackware](https://docs.slackware.com/howtos:slackware_admin:kernelbuilding) + +### Mount `debugfs` and `tracefs` + +The eBPF collector also requires both the `tracefs` and `debugfs` filesystems. Try mounting the `tracefs` and `debugfs` +filesystems using the commands below: + +```bash +sudo mount -t debugfs nodev /sys/kernel/debug +sudo mount -t tracefs nodev /sys/kernel/tracing +``` +​ +If they are already mounted, you will see an error. If they are not mounted, they should be after running those two +commands. You can also configure your system's `/etc/fstab` configuration to mount these filesystems. + +### Install Netdata with the `--enable-ebpf` + +eBPF collection is only enabled if you install Netdata with the `--enable-ebpf` option. + +If you installed via the [one-line installation script](/packaging/installer/README.md), [64-bit +binary](/packaging/installer/methods/kickstart-64.md), or [manually](/packaging/installer/methods/manual.md), you can +append the `--enable-ebpf` option when you reinstall. + +For example, if you used the one-line installation script, you can reinstall Netdata with the following: + +```bash +bash <(curl -Ss https://my-netdata.io/kickstart.sh) --enable-ebpf +``` + +This process will not overwrite any changes you made to configuration files. + +### Edit `netdata.conf` to enable the collector + +After installing Netdata with the `--enable-ebpf` option, you still need to enable the plugin explicitly. To do so, use +`edit-config` to open `netdata.conf` and set `ebpf = yes` in the `[plugins]` section. + +```bash +cd /etc/netdata/ # Replace with your Netdata configuration directory, if not /etc/netdata/ +./edit-config netdata.conf +``` + +Scroll down to the `[plugins]` section and uncomment the `ebpf` line after changing its setting to `yes`. + +```conf +[plugins] + ebpf = yes +``` + +Restart Netdata with `service netdata restart`, or the appropriate method for your system, and reload your browser to +see eBPF charts. + +## Charts + +The first version of `ebpf.plugin` gives a general vision about process running on computer. The charts related +to this plugin are inside the **eBPF** option on dashboard menu and divided in three groups `file`, `vfs`, and +`process`. + +All the collector charts show values per second. The collector retains the total value, but charts only show the +difference between the previous and current metrics collections. + +### File + +This group has two charts to demonstrate how software interacts with the Linux kernel to open and close file +descriptors. + +#### File descriptor + +This chart contains two dimensions that show the number of calls to the functions `do_sys_open` and `__close_fd`. These +functions are not commonly called from software, but they are behind the system cals `open(2)`, `openat(2)`, and +`close(2)`. ​ + +#### File error + +This charts demonstrate the number of times some software tried and failed to open or close a file descriptor. + +### VFS + +A [virtual file system](https://en.wikipedia.org/wiki/Virtual_file_system) (VFS) is a layer on top of regular +filesystems. The functions present inside this API are used for all filesystems, so it's possible the charts in this +group won't show _all_ the actions that occured on your system. + +#### Deleted objects + +This chart monitors calls for `vfs_unlink`. This function is responsible for removing object from the file system. + +#### IO + +This chart shows the number of calls to the functions `vfs_read` and `vfs_write`. + +#### IO bytes + +This chart also monitors `vfs_read` and `vfs_write`, but instead shows the total of bytes read and written with these +functions. + +Netdata displays the number of bytes written as negative, because they are moving down to disk. + +#### IO errors + +Netdata counts and shows the number of instances where a running program experiences a read or write error. + +### Process + +For this group, the eBPF collector monitors process/thread creation and process end, and then displays any errors in the +following charts. + +#### Process thread + +Internally, the Linux kernel treats both process and threads as `tasks`. To create a thread, the kernel offers a few +system calls: `fork(2)`, `vfork(2)` and `clone(2)`. Each of these system calls in turn use the function `_do_fork`. To +generate this chart, Netdata monitors `_do_fork` to populate the `process` dimension, and monitors `sys_clone` to +identify threads + +#### Exit + +Ending a task is actually two steps. The first is a call to the internal function `do_exit`, which notifies the +operating system that the task is finishing its work. The second step is the release of kernel information, which is +done with the internal function `release_task`. The difference between the two dimensions can help you discover [zombie +processes](https://en.wikipedia.org/wiki/Zombie_process). + +#### Task error + +The functions responsible for ending tasks do not return values, so this chart contains information about failures on +process and thread creation. + +## Configuration + +This plugin has different configuration modes, all of which can be adjusted with its configuration file at +`ebpf.conf`. By default, the plugin uses the less expensive `entry` mode. You can learn more about how the +plugin works using `entry` by reading this configuration file. + +You can always edit this file with `edit-config`: + +```bash +cd /etc/netdata/ # Replace with your Netdata configuration directory, if not /etc/netdata/ +./edit-config ebpf.conf +``` + +### `[global]` + +In this section we define variables applied to the whole collector and the other subsections. + +#### load + +The collector has two different eBPF programs. These programs monitor the same functions inside the kernel, but they +monitor, process, and display different kinds of information. + +By default, this plugin uses the `entry` mode. Changing this mode can create significant overhead on your operating +system, but also offer important information if you are developing or debugging software. The `load` option accepts the +following values: ​ + +- `entry`: This is the default mode. In this mode, Netdata monitors only calls for the functions described in the + sections above. When this mode is selected, Netdata does not show charts related to errors. +- `return`: In this mode, Netdata also monitors the calls to function. In the `entry` mode, Netdata only traces kernel + functions, but with `return`, Netdata also monitors the return of each function. This mode creates more charts, but + also creates an overhead of roughly 110 nanosections for each function call. + +## Performance + +Because eBPF monitoring is complex, we are evaluating the performance of this new collector in various real-world +conditions, across various system loads, and when monitoring complex applications. + +Our [initial testing](https://github.com/netdata/netdata/issues/8195) shows the performance of the eBPF collector is +nearly identical to our [apps.plugin collector](/collectors/apps.plugin/README.md), despite collecting and displaying +much more sophisticated metrics. You can now use the eBPF to gather deeper insights without affecting the performance of +your complex applications at any load. diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c new file mode 100644 index 0000000000..b3fa7f71bf --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.c @@ -0,0 +1,1152 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include + +#include "ebpf.h" + +// callback required by eval() +int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result) { + (void)variable; + (void)hash; + (void)rc; + (void)result; + return 0; +}; + +void send_statistics( const char *action, const char *action_result, const char *action_data) { + (void) action; + (void) action_result; + (void) action_data; + return; +} + +// callbacks required by popen() +void signals_block(void) {}; +void signals_unblock(void) {}; +void signals_reset(void) {}; + +// required by get_system_cpus() +char *netdata_configured_host_prefix = ""; + +// callback required by fatal() +void netdata_cleanup_and_exit(int ret) { + exit(ret); +} + +// ---------------------------------------------------------------------- +//Netdata eBPF library +void *libnetdata = NULL; +int (*load_bpf_file)(char *, int) = NULL; +int (*set_bpf_perf_event)(int, int); +int (*perf_event_unmap)(struct perf_event_mmap_page *, size_t); +int (*perf_event_mmap_header)(int, struct perf_event_mmap_page **, int); +void (*netdata_perf_loop_multi)(int *, struct perf_event_mmap_page **, int, int *, int (*nsb)(void *, int), int); +int *map_fd = NULL; + +//Perf event variables +static int pmu_fd[NETDATA_MAX_PROCESSOR]; +static struct perf_event_mmap_page *headers[NETDATA_MAX_PROCESSOR]; +int page_cnt = 8; + +//Libbpf (It is necessary to have at least kernel 4.10) +int (*bpf_map_lookup_elem)(int, const void *, void *); + +static char *plugin_dir = PLUGINS_DIR; +static char *user_config_dir = CONFIG_DIR; +static char *stock_config_dir = LIBCONFIG_DIR; +static char *netdata_configured_log_dir = LOG_DIR; + +FILE *developer_log = NULL; + +//Global vectors +netdata_syscall_stat_t *aggregated_data = NULL; +netdata_publish_syscall_t *publish_aggregated = NULL; + +static int update_every = 1; +static int thread_finished = 0; +static int close_plugin = 0; +static netdata_run_mode_t mode = MODE_ENTRY; +static int debug_log = 0; +static int use_stdout = 0; +struct config collector_config; +static int mykernel = 0; +static int nprocs; +static int isrh; +netdata_idx_t *hash_values; + +pthread_mutex_t lock; + +static struct ebpf_module { + const char *thread_name; + int enabled; + void (*start_routine) (void *); + int update_time; + int global_charts; + int apps_charts; + netdata_run_mode_t mode; +} ebpf_modules[] = { + { .thread_name = "process", .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY }, + { .thread_name = "network_viewer", .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY }, + { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 0, .apps_charts = 1, .mode = MODE_ENTRY }, +}; + +static char *dimension_names[NETDATA_MAX_MONITOR_VECTOR] = { "open", "close", "delete", "read", "write", "process", "task", "process", "thread" }; +static char *id_names[NETDATA_MAX_MONITOR_VECTOR] = { "do_sys_open", "__close_fd", "vfs_unlink", "vfs_read", "vfs_write", "do_exit", "release_task", "_do_fork", "sys_clone" }; +static char *status[] = { "process", "zombie" }; + +int event_pid = 0; +netdata_ebpf_events_t collector_events[] = { + { .type = 'r', .name = "vfs_write" }, + { .type = 'r', .name = "vfs_writev" }, + { .type = 'r', .name = "vfs_read" }, + { .type = 'r', .name = "vfs_readv" }, + { .type = 'r', .name = "do_sys_open" }, + { .type = 'r', .name = "vfs_unlink" }, + { .type = 'p', .name = "do_exit" }, + { .type = 'p', .name = "release_task" }, + { .type = 'r', .name = "_do_fork" }, + { .type = 'r', .name = "__close_fd" }, + { .type = 'r', .name = "__x64_sys_clone" }, + { .type = 0, .name = NULL } +}; + +void open_developer_log() { + char filename[FILENAME_MAX+1]; + int tot = sprintf(filename, "%s/%s", netdata_configured_log_dir, NETDATA_DEVELOPER_LOG_FILE); + + if(tot > 0) + developer_log = fopen(filename, "a"); +} + +static int unmap_memory() { + int i; + int size = (int)sysconf(_SC_PAGESIZE)*(page_cnt + 1); + for ( i = 0 ; i < nprocs ; i++ ) { + if (perf_event_unmap(headers[i], size) < 0) { + fprintf(stderr,"[EBPF PROCESS] CANNOT unmap headers.\n"); + return -1; + } + + close(pmu_fd[i]); + } + + return 0; +} + +static void int_exit(int sig) +{ + close_plugin = 1; + + //When both threads were not finished case I try to go in front this address, the collector will crash + if (!thread_finished) { + return; + } + + if (aggregated_data) { + free(aggregated_data); + aggregated_data = NULL; + } + + if (publish_aggregated) { + free(publish_aggregated); + publish_aggregated = NULL; + } + + if(mode == MODE_DEVMODE && debug_log) { + unmap_memory(); + } + + if (libnetdata) { + dlclose(libnetdata); + libnetdata = NULL; + } + + if (developer_log) { + fclose(developer_log); + developer_log = NULL; + } + + if (hash_values) { + freez(hash_values); + } + + if (event_pid) { + int ret = fork(); + if (ret < 0) //error + error("[EBPF PROCESS] Cannot fork(), so I won't be able to clean %skprobe_events", NETDATA_DEBUGFS); + else if (!ret) { //child + int i; + for ( i=getdtablesize(); i>=0; --i) + close(i); + + int fd = open("/dev/null",O_RDWR, 0); + if (fd != -1) { + dup2 (fd, STDIN_FILENO); + dup2 (fd, STDOUT_FILENO); + dup2 (fd, STDERR_FILENO); + } + + if (fd > 2) + close (fd); + + int sid = setsid(); + if(sid >= 0) { + sleep(1); + if(debug_log) { + open_developer_log(); + } + debug(D_EXIT, "Wait for father %d die", event_pid); + clean_kprobe_events(developer_log, event_pid, collector_events); + } else { + error("Cannot become session id leader, so I won't try to clean kprobe_events.\n"); + } + } else { //parent + exit(0); + } + + if (developer_log) { + fclose(developer_log); + developer_log = NULL; + } + } + + exit(sig); +} + +static inline void netdata_write_chart_cmd(char *type + , char *id + , char *axis + , char *web + , int order) +{ + printf("CHART %s.%s '' '' '%s' '%s' '' line %d 1 ''\n" + , type + , id + , axis + , web + , order); +} + +static void netdata_write_global_dimension(char *d, char *n) +{ + printf("DIMENSION %s %s absolute 1 1\n", d, n); +} + +static void netdata_create_global_dimension(void *ptr, int end) +{ + netdata_publish_syscall_t *move = ptr; + + int i = 0; + while (move && i < end) { + netdata_write_global_dimension(move->name, move->dimension); + + move = move->next; + i++; + } +} +static inline void netdata_create_chart(char *family + , char *name + , char *axis + , char *web + , int order + , void (*ncd)(void *, int) + , void *move + , int end) +{ + netdata_write_chart_cmd(family, name, axis, web, order); + + ncd(move, end); +} + +static void netdata_create_io_chart(char *family, char *name, char *axis, char *web, int order) { + printf("CHART %s.%s '' '' '%s' '%s' '' line %d 1 ''\n" + , family + , name + , axis + , web + , order); + + printf("DIMENSION %s %s absolute 1 1\n", id_names[3], NETDATA_VFS_DIM_OUT_FILE_BYTES); + printf("DIMENSION %s %s absolute 1 1\n", id_names[4], NETDATA_VFS_DIM_IN_FILE_BYTES); +} + +static void netdata_process_status_chart(char *family, char *name, char *axis, char *web, int order) { + printf("CHART %s.%s '' '' '%s' '%s' '' line %d 1 ''\n" + , family + , name + , axis + , web + , order); + + printf("DIMENSION %s '' absolute 1 1\n", status[0]); + printf("DIMENSION %s '' absolute 1 1\n", status[1]); +} + +static void netdata_global_charts_create() { + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_FILE_OPEN_CLOSE_COUNT + , "Calls" + , NETDATA_FILE_GROUP + , 970 + , netdata_create_global_dimension + , publish_aggregated + , 2); + + if(mode < MODE_ENTRY) { + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_FILE_OPEN_ERR_COUNT + , "Calls" + , NETDATA_FILE_GROUP + , 971 + , netdata_create_global_dimension + , publish_aggregated + , 2); + } + + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_VFS_FILE_CLEAN_COUNT + , "Calls" + , NETDATA_VFS_GROUP + , 972 + , netdata_create_global_dimension + , &publish_aggregated[NETDATA_DEL_START] + , 1); + + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_VFS_FILE_IO_COUNT + , "Calls" + , NETDATA_VFS_GROUP + , 973 + , netdata_create_global_dimension + , &publish_aggregated[NETDATA_IN_START_BYTE] + , 2); + + if(mode < MODE_ENTRY) { + netdata_create_io_chart(NETDATA_EBPF_FAMILY + , NETDATA_VFS_IO_FILE_BYTES + , "bytes/s" + , NETDATA_VFS_GROUP + , 974); + + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_VFS_FILE_ERR_COUNT + , "Calls" + , NETDATA_VFS_GROUP + , 975 + , netdata_create_global_dimension + , &publish_aggregated[2] + , NETDATA_VFS_ERRORS); + + } + + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_PROCESS_SYSCALL + , "Calls" + , NETDATA_PROCESS_GROUP + , 976 + , netdata_create_global_dimension + , &publish_aggregated[NETDATA_PROCESS_START] + , 2); + + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_EXIT_SYSCALL + , "Calls" + , NETDATA_PROCESS_GROUP + , 977 + , netdata_create_global_dimension + , &publish_aggregated[NETDATA_EXIT_START] + , 2); + + netdata_process_status_chart(NETDATA_EBPF_FAMILY + , NETDATA_PROCESS_STATUS_NAME + , "Total" + , NETDATA_PROCESS_GROUP + , 978); + + if(mode < MODE_ENTRY) { + netdata_create_chart(NETDATA_EBPF_FAMILY + , NETDATA_PROCESS_ERROR_NAME + , "Calls" + , NETDATA_PROCESS_GROUP + , 979 + , netdata_create_global_dimension + , &publish_aggregated[NETDATA_PROCESS_START] + , 2); + } + +} + + +static void netdata_create_charts() { + netdata_global_charts_create(); +} + +static void netdata_update_publish(netdata_publish_syscall_t *publish + , netdata_publish_vfs_common_t *pvc + , netdata_syscall_stat_t *input) { + + netdata_publish_syscall_t *move = publish; + while(move) { + if(input->call != move->pcall) { + //This condition happens to avoid initial values with dimensions higher than normal values. + if(move->pcall) { + move->ncall = (input->call > move->pcall)?input->call - move->pcall: move->pcall - input->call; + move->nbyte = (input->bytes > move->pbyte)?input->bytes - move->pbyte: move->pbyte - input->bytes; + move->nerr = (input->ecall > move->nerr)?input->ecall - move->perr: move->perr - input->ecall; + } else { + move->ncall = 0; + move->nbyte = 0; + move->nerr = 0; + } + + move->pcall = input->call; + move->pbyte = input->bytes; + move->perr = input->ecall; + } else { + move->ncall = 0; + move->nbyte = 0; + move->nerr = 0; + } + + input = input->next; + move = move->next; + } + + pvc->write = -((long)publish[2].nbyte); + pvc->read = (long)publish[3].nbyte; + + pvc->running = (long)publish[7].ncall - (long)publish[8].ncall; + publish[6].ncall = -publish[6].ncall; // release + pvc->zombie = (long)publish[5].ncall + (long)publish[6].ncall; +} + +static inline void write_begin_chart(char *family, char *name) +{ + int ret = printf( "BEGIN %s.%s\n" + , family + , name); + + (void)ret; +} + +static inline void write_chart_dimension(char *dim, long long value) +{ + int ret = printf("SET %s = %lld\n", dim, value); + (void)ret; +} + +static void write_global_count_chart(char *name, char *family, netdata_publish_syscall_t *move, int end) { + write_begin_chart(family, name); + + int i = 0; + while (move && i < end) { + write_chart_dimension(move->name, move->ncall); + + move = move->next; + i++; + } + + printf("END\n"); +} + +static void write_global_err_chart(char *name, char *family, netdata_publish_syscall_t *move, int end) { + write_begin_chart(family, name); + + int i = 0; + while (move && i < end) { + write_chart_dimension(move->name, move->nerr); + + move = move->next; + i++; + } + + printf("END\n"); +} + +static void write_io_chart(char *family, netdata_publish_vfs_common_t *pvc) { + write_begin_chart(family, NETDATA_VFS_IO_FILE_BYTES); + + write_chart_dimension(id_names[3], (long long) pvc->write); + write_chart_dimension(id_names[4], (long long) pvc->read); + + printf("END\n"); +} + +static void write_status_chart(char *family, netdata_publish_vfs_common_t *pvc) { + write_begin_chart(family, NETDATA_PROCESS_STATUS_NAME); + + write_chart_dimension(status[0], (long long) pvc->running); + write_chart_dimension(status[1], (long long) pvc->zombie); + + printf("END\n"); +} + +static void netdata_publish_data() { + netdata_publish_vfs_common_t pvc; + netdata_update_publish(publish_aggregated, &pvc, aggregated_data); + + write_global_count_chart(NETDATA_FILE_OPEN_CLOSE_COUNT, NETDATA_EBPF_FAMILY, publish_aggregated, 2); + write_global_count_chart(NETDATA_VFS_FILE_CLEAN_COUNT, NETDATA_EBPF_FAMILY, &publish_aggregated[NETDATA_DEL_START], 1); + write_global_count_chart(NETDATA_VFS_FILE_IO_COUNT, NETDATA_EBPF_FAMILY, &publish_aggregated[NETDATA_IN_START_BYTE], 2); + write_global_count_chart(NETDATA_EXIT_SYSCALL, NETDATA_EBPF_FAMILY, &publish_aggregated[NETDATA_EXIT_START], 2); + write_global_count_chart(NETDATA_PROCESS_SYSCALL, NETDATA_EBPF_FAMILY, &publish_aggregated[NETDATA_PROCESS_START], 2); + + write_status_chart(NETDATA_EBPF_FAMILY, &pvc); + if(mode < MODE_ENTRY) { + write_global_err_chart(NETDATA_FILE_OPEN_ERR_COUNT, NETDATA_EBPF_FAMILY, publish_aggregated, 2); + write_global_err_chart(NETDATA_VFS_FILE_ERR_COUNT, NETDATA_EBPF_FAMILY, &publish_aggregated[2], NETDATA_VFS_ERRORS); + write_global_err_chart(NETDATA_PROCESS_ERROR_NAME, NETDATA_EBPF_FAMILY, &publish_aggregated[NETDATA_PROCESS_START], 2); + + write_io_chart(NETDATA_EBPF_FAMILY, &pvc); + } +} + +void *process_publisher(void *ptr) +{ + (void)ptr; + netdata_create_charts(); + + usec_t step = update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); + while(!close_plugin) { + usec_t dt = heartbeat_next(&hb, step); + (void)dt; + + pthread_mutex_lock(&lock); + netdata_publish_data(); + pthread_mutex_unlock(&lock); + + fflush(stdout); + } + + return NULL; +} + +static void move_from_kernel2user_global() { + uint64_t idx; + netdata_idx_t res[NETDATA_GLOBAL_VECTOR]; + + netdata_idx_t *val = hash_values; + for (idx = 0; idx < NETDATA_GLOBAL_VECTOR; idx++) { + if(!bpf_map_lookup_elem(map_fd[1], &idx, val)) { + uint64_t total = 0; + int i; + int end = (mykernel < NETDATA_KERNEL_V4_15)?1:nprocs; + for (i = 0; i < end; i++) + total += val[i]; + + res[idx] = total; + } else { + res[idx] = 0; + } + } + + aggregated_data[0].call = res[NETDATA_KEY_CALLS_DO_SYS_OPEN]; + aggregated_data[1].call = res[NETDATA_KEY_CALLS_CLOSE_FD]; + aggregated_data[2].call = res[NETDATA_KEY_CALLS_VFS_UNLINK]; + aggregated_data[3].call = res[NETDATA_KEY_CALLS_VFS_READ] + res[NETDATA_KEY_CALLS_VFS_READV]; + aggregated_data[4].call = res[NETDATA_KEY_CALLS_VFS_WRITE] + res[NETDATA_KEY_CALLS_VFS_WRITEV]; + aggregated_data[5].call = res[NETDATA_KEY_CALLS_DO_EXIT]; + aggregated_data[6].call = res[NETDATA_KEY_CALLS_RELEASE_TASK]; + aggregated_data[7].call = res[NETDATA_KEY_CALLS_DO_FORK]; + aggregated_data[8].call = res[NETDATA_KEY_CALLS_SYS_CLONE]; + + aggregated_data[0].ecall = res[NETDATA_KEY_ERROR_DO_SYS_OPEN]; + aggregated_data[1].ecall = res[NETDATA_KEY_ERROR_CLOSE_FD]; + aggregated_data[2].ecall = res[NETDATA_KEY_ERROR_VFS_UNLINK]; + aggregated_data[3].ecall = res[NETDATA_KEY_ERROR_VFS_READ] + res[NETDATA_KEY_ERROR_VFS_READV]; + aggregated_data[4].ecall = res[NETDATA_KEY_ERROR_VFS_WRITE] + res[NETDATA_KEY_ERROR_VFS_WRITEV]; + aggregated_data[7].ecall = res[NETDATA_KEY_ERROR_DO_FORK]; + aggregated_data[8].ecall = res[NETDATA_KEY_ERROR_SYS_CLONE]; + + aggregated_data[2].bytes = (uint64_t)res[NETDATA_KEY_BYTES_VFS_WRITE] + (uint64_t)res[NETDATA_KEY_BYTES_VFS_WRITEV]; + aggregated_data[3].bytes = (uint64_t)res[NETDATA_KEY_BYTES_VFS_READ] + (uint64_t)res[NETDATA_KEY_BYTES_VFS_READV]; +} + +static void move_from_kernel2user() +{ + move_from_kernel2user_global(); +} + +void *process_collector(void *ptr) +{ + (void)ptr; + + usec_t step = 778879ULL; + heartbeat_t hb; + heartbeat_init(&hb); + while(!close_plugin) { + usec_t dt = heartbeat_next(&hb, step); + (void)dt; + + pthread_mutex_lock(&lock); + move_from_kernel2user(); + pthread_mutex_unlock(&lock); + } + + return NULL; +} + +static int netdata_store_bpf(void *data, int size) { + (void)size; + + if (close_plugin) + return 0; + + if(!debug_log) + return -2; //LIBBPF_PERF_EVENT_CONT; + + netdata_error_report_t *e = data; + fprintf(developer_log + ,"%llu %s %u: %s, %d\n" + , now_realtime_usec() ,e->comm, e->pid, dimension_names[e->type], e->err); + fflush(developer_log); + + return -2; //LIBBPF_PERF_EVENT_CONT; +} + +void *process_log(void *ptr) +{ + (void) ptr; + + if (mode == MODE_DEVMODE && debug_log) { + netdata_perf_loop_multi(pmu_fd, headers, nprocs, &close_plugin, netdata_store_bpf, page_cnt); + } + + return NULL; +} + +void set_global_labels() { + int i; + + netdata_syscall_stat_t *is = aggregated_data; + netdata_syscall_stat_t *prev = NULL; + + netdata_publish_syscall_t *pio = publish_aggregated; + netdata_publish_syscall_t *publish_prev = NULL; + for (i = 0; i < NETDATA_MAX_MONITOR_VECTOR; i++) { + if(prev) { + prev->next = &is[i]; + } + prev = &is[i]; + + pio[i].dimension = dimension_names[i]; + pio[i].name = id_names[i]; + if(publish_prev) { + publish_prev->next = &pio[i]; + } + publish_prev = &pio[i]; + } +} + +int allocate_global_vectors() { + aggregated_data = callocz(NETDATA_MAX_MONITOR_VECTOR, sizeof(netdata_syscall_stat_t)); + if(!aggregated_data) { + return -1; + } + + publish_aggregated = callocz(NETDATA_MAX_MONITOR_VECTOR, sizeof(netdata_publish_syscall_t)); + if(!publish_aggregated) { + return -1; + } + + hash_values = callocz(nprocs, sizeof(netdata_idx_t)); + if(!hash_values) { + return -1; + } + + return 0; +} + +static void build_complete_path(char *out, size_t length,char *path, char *filename) { + if(path){ + snprintf(out, length, "%s/%s", path, filename); + } else { + snprintf(out, length, "%s", filename); + } +} + +static int map_memory() { + int i; + for (i = 0; i < nprocs; i++) { + pmu_fd[i] = set_bpf_perf_event(i, 2); + + if (perf_event_mmap_header(pmu_fd[i], &headers[i], page_cnt) < 0) { + return -1; + } + } + return 0; +} + +static int ebpf_load_libraries() +{ + char *err = NULL; + char lpath[4096]; + + build_complete_path(lpath, 4096, plugin_dir, "libnetdata_ebpf.so"); + libnetdata = dlopen(lpath, RTLD_LAZY); + if (!libnetdata) { + error("[EBPF_PROCESS] Cannot load %s.", lpath); + return -1; + } else { + load_bpf_file = dlsym(libnetdata, "load_bpf_file"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find load_bpf_file: %s", err); + return -1; + } + + map_fd = dlsym(libnetdata, "map_fd"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find map_fd: %s", err); + return -1; + } + + bpf_map_lookup_elem = dlsym(libnetdata, "bpf_map_lookup_elem"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find bpf_map_lookup_elem: %s", err); + return -1; + } + + if(mode == MODE_DEVMODE) { + set_bpf_perf_event = dlsym(libnetdata, "set_bpf_perf_event"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find set_bpf_perf_event: %s", err); + return -1; + } + + perf_event_unmap = dlsym(libnetdata, "perf_event_unmap"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find perf_event_unmap: %s", err); + return -1; + } + + perf_event_mmap_header = dlsym(libnetdata, "perf_event_mmap_header"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find perf_event_mmap_header: %s", err); + return -1; + } + + netdata_perf_loop_multi = dlsym(libnetdata, "netdata_perf_loop_multi"); + if ((err = dlerror()) != NULL) { + error("[EBPF_PROCESS] Cannot find netdata_perf_loop_multi: %s", err); + return -1; + } + } + } + + return 0; +} + +char *select_file() { + if(!mode) + return "rnetdata_ebpf_process.o"; + if(mode == MODE_DEVMODE) + return "dnetdata_ebpf_process.o"; + + return "pnetdata_ebpf_process.o"; +} + +int process_load_ebpf() +{ + char lpath[4096]; + + char *name = select_file(); + + build_complete_path(lpath, 4096, plugin_dir, name); + event_pid = getpid(); + if (load_bpf_file(lpath, event_pid) ) { + error("[EBPF_PROCESS] Cannot load program: %s", lpath); + return -1; + } else { + info("[EBPF PROCESS]: The eBPF program %s was loaded with success.", name); + } + + return 0; +} + +void set_global_variables() { + //Get environment variables + plugin_dir = getenv("NETDATA_PLUGINS_DIR"); + if(!plugin_dir) + plugin_dir = PLUGINS_DIR; + + user_config_dir = getenv("NETDATA_USER_CONFIG_DIR"); + if(!user_config_dir) + user_config_dir = CONFIG_DIR; + + stock_config_dir = getenv("NETDATA_STOCK_CONFIG_DIR"); + if(!stock_config_dir) + stock_config_dir = LIBCONFIG_DIR; + + netdata_configured_log_dir = getenv("NETDATA_LOG_DIR"); + if(!netdata_configured_log_dir) + netdata_configured_log_dir = LOG_DIR; + + page_cnt *= (int)sysconf(_SC_NPROCESSORS_ONLN); + + nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (nprocs > NETDATA_MAX_PROCESSOR) { + nprocs = NETDATA_MAX_PROCESSOR; + } + + isrh = get_redhat_release(); +} + +static void change_collector_event() { + int i; + if (mykernel < NETDATA_KERNEL_V5_3) + collector_events[10].name = NULL; + + for (i = 0; collector_events[i].name ; i++ ) { + collector_events[i].type = 'p'; + } +} + +static void change_syscalls() { + static char *lfork = { "do_fork" }; + id_names[7] = lfork; + collector_events[8].name = lfork; +} + +static inline void what_to_load(char *ptr) { + if (!strcasecmp(ptr, "return")) + mode = MODE_RETURN; + /* + else if (!strcasecmp(ptr, "dev")) + mode = 1; + */ + else + change_collector_event(); + + if (isrh >= NETDATA_MINIMUM_RH_VERSION && isrh < NETDATA_RH_8) + change_syscalls(); +} + +static inline void enable_debug(char *ptr) { + if (!strcasecmp(ptr, "yes")) + debug_log = 1; +} + +static inline void set_log_file(char *ptr) { + if (!strcasecmp(ptr, "yes")) + use_stdout = 1; +} + +static void set_global_values() { + struct section *sec = collector_config.first_section; + while(sec) { + if(!strcasecmp(sec->name, "global")) { + struct config_option *values = sec->values; + while(values) { + if(!strcasecmp(values->name, "load")) + what_to_load(values->value); + else if(!strcasecmp(values->name, "debug log")) + enable_debug(values->value); + else if(!strcasecmp(values->name, "use stdout")) + set_log_file(values->value); + + values = values->next; + } + } + sec = sec->next; + } +} + +static int load_collector_file(char *path) { + char lpath[4096]; + + build_complete_path(lpath, 4096, path, "ebpf.conf" ); + + if (!appconfig_load(&collector_config, lpath, 0, NULL)) + return 1; + + set_global_values(); + + return 0; +} + +static inline void ebpf_disable_apps() { + int i ; + for (i = 0 ;ebpf_modules[i].thread_name ; i++ ) { + ebpf_modules[i].apps_charts = 0; + } +} + +static inline void ebpf_enable_specific_chart(struct ebpf_module *em, int disable_apps) { + em->enabled = 1; + if (!disable_apps) { + em->apps_charts = 1; + } + em->global_charts = 1; +} + +static inline void ebpf_enable_all_charts(int apps) { + int i ; + for (i = 0 ; ebpf_modules[i].thread_name ; i++ ) { + ebpf_enable_specific_chart(&ebpf_modules[i], apps); + } +} + +static inline void ebpf_enable_chart(int enable, int disable_apps) { + int i ; + for (i = 0 ; ebpf_modules[i].thread_name ; i++ ) { + if (i == enable) { + ebpf_enable_specific_chart(&ebpf_modules[i], disable_apps); + break; + } + } +} + +static inline void ebpf_set_thread_mode(netdata_run_mode_t lmode) { + int i ; + for (i = 0 ; ebpf_modules[i].thread_name ; i++ ) { + ebpf_modules[i].mode = lmode; + } +} + +void ebpf_print_help() { + const time_t t = time(NULL); + struct tm ct; + struct tm *test = localtime_r(&t, &ct); + int year; + if (test) + year = ct.tm_year; + else + year = 0; + + fprintf(stderr, + "\n" + " Netdata ebpf.plugin %s\n" + " Copyright (C) 2016-%d Costa Tsaousis \n" + " Released under GNU General Public License v3 or later.\n" + " All rights reserved.\n" + "\n" + " This program is a data collector plugin for netdata.\n" + "\n" + " Available command line options:\n" + "\n" + " SECONDS set the data collection frequency.\n" + "\n" + " --help or -h show this help.\n" + "\n" + " --version or -v show software version.\n" + "\n" + " --global or -g disable charts per application.\n" + "\n" + " --all or -a Enable all chart groups (global and apps), unless -g is also given.\n" + "\n" + " --net or -n Enable network viewer charts.\n" + "\n" + " --process or -p Enable charts related to process run time.\n" + "\n" + " --return or -r Run the collector in return mode.\n" + "\n" + , VERSION + , (year >= 116)?year + 1900: 2020 + ); +} + +static void parse_args(int argc, char **argv) +{ + int enabled = 0; + int disable_apps = 0; + int freq = 0; + int c; + int option_index = 0; + static struct option long_options[] = { + {"help", no_argument, 0, 'h' }, + {"version", no_argument, 0, 'v' }, + {"global", no_argument, 0, 'g' }, + {"all", no_argument, 0, 'a' }, + {"net", no_argument, 0, 'n' }, + {"process", no_argument, 0, 'p' }, + {"return", no_argument, 0, 'r' }, + {0, 0, 0, 0} + }; + + if (argc > 1) { + int n = (int)str2l(argv[1]); + if(n > 0) { + freq = n; + } + } + + while (1) { + c = getopt_long(argc, argv, "hvganpr",long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 'h': { + ebpf_print_help(); + exit(0); + } + case 'v': { + printf("ebpf.plugin %s\n", VERSION); + exit(0); + } + case 'g': { + disable_apps = 1; + ebpf_disable_apps(); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF running with global chart group, because it was started with the option \"--global\" or \"-g\"."); +#endif + break; + } + case 'a': { + ebpf_enable_all_charts(disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF running with all chart groups, because it was started with the option \"--all\" or \"-a\"."); +#endif + break; + } + case 'n': { + enabled = 1; + ebpf_enable_chart(1, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF enabling \"NET\" charts, because it was started with the option \"--net\" or \"-n\"."); +#endif + break; + } + case 'p': { + enabled = 1; + ebpf_enable_chart(0, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF enabling \"PROCESS\" charts, because it was started with the option \"--process\" or \"-p\"."); +#endif + break; + } + case 'r': { + mode = MODE_RETURN; + ebpf_set_thread_mode(mode); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF running in \"return\" mode, because it was started with the option \"--return\" or \"-r\"."); +#endif + break; + } + default: { + break; + } + } + } + + if (freq > 0) { + update_every = freq; + } + + if (!enabled) { + ebpf_enable_all_charts(disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF running with all charts, because neither \"-n\" or \"-p\" was given."); +#endif + } +} + +int main(int argc, char **argv) +{ + parse_args(argc, argv); + + mykernel = get_kernel_version(); + if(!has_condition_to_run(mykernel)) { + error("[EBPF PROCESS] The current collector cannot run on this kernel."); + return 1; + } + + //set name + program_name = "ebpf.plugin"; + + //disable syslog + error_log_syslog = 0; + + // set errors flood protection to 100 logs per hour + error_log_errors_per_period = 100; + error_log_throttle_period = 3600; + + struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; + if (setrlimit(RLIMIT_MEMLOCK, &r)) { + error("[EBPF PROCESS] setrlimit(RLIMIT_MEMLOCK)"); + return 2; + } + + set_global_variables(); + + if (load_collector_file(user_config_dir)) { + info("[EBPF PROCESS] does not have a configuration file. It is starting with default options."); + change_collector_event(); + if (isrh >= NETDATA_MINIMUM_RH_VERSION && isrh < NETDATA_RH_8) + change_syscalls(); + } + + if(ebpf_load_libraries()) { + error("[EBPF_PROCESS] Cannot load library."); + thread_finished++; + int_exit(3); + } + + signal(SIGINT, int_exit); + signal(SIGTERM, int_exit); + + if (process_load_ebpf()) { + thread_finished++; + int_exit(4); + } + + if(allocate_global_vectors()) { + thread_finished++; + error("[EBPF_PROCESS] Cannot allocate necessary vectors."); + int_exit(5); + } + + if(mode == MODE_DEVMODE && debug_log) { + if(map_memory()) { + thread_finished++; + error("[EBPF_PROCESS] Cannot map memory used with perf events."); + int_exit(6); + } + } + + set_global_labels(); + + if(debug_log) { + open_developer_log(); + } + + if (pthread_mutex_init(&lock, NULL)) { + thread_finished++; + error("[EBPF PROCESS] Cannot start the mutex."); + int_exit(7); + } + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + pthread_t thread[NETDATA_EBPF_PROCESS_THREADS]; + + int i; + int end = NETDATA_EBPF_PROCESS_THREADS; + + void * (*function_pointer[])(void *) = {process_publisher, process_collector, process_log }; + + for ( i = 0; i < end ; i++ ) { + if ( ( pthread_create(&thread[i], &attr, function_pointer[i], NULL) ) ) { + error("[EBPF_PROCESS] Cannot create threads."); + thread_finished++; + int_exit(8); + } + } + + for ( i = 0; i < end ; i++ ) { + if ( (pthread_join(thread[i], NULL) ) ) { + error("[EBPF_PROCESS] Cannot join threads."); + thread_finished++; + int_exit(9); + } + } + + thread_finished++; + int_exit(0); + + return 0; +} diff --git a/collectors/ebpf.plugin/ebpf.conf b/collectors/ebpf.plugin/ebpf.conf new file mode 100644 index 0000000000..88d7050c7b --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.conf @@ -0,0 +1,2 @@ +[global] + load = entry diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h new file mode 100644 index 0000000000..5f05389e02 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.h @@ -0,0 +1,155 @@ +#ifndef _NETDATA_VFS_EBPF_H_ +# define _NETDATA_VFS_EBPF_H_ 1 + +# include + +#ifndef __FreeBSD__ +# include +# endif +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# define NETDATA_GLOBAL_VECTOR 24 +# define NETDATA_MAX_MONITOR_VECTOR 9 +# define NETDATA_VFS_ERRORS 3 +# define NETDATA_PROCESS_ERRORS 4 + +# define NETDATA_DEL_START 2 +# define NETDATA_IN_START_BYTE 3 +# define NETDATA_EXIT_START 5 +# define NETDATA_PROCESS_START 7 +# define NETDATA_PROCESS_RUNNING_COUNT 9 + +# define NETDATA_EBPF_PROCESS_THREADS (uint32_t)3 + +# include +# include +# include + +//From libnetdata.h +# include "../../libnetdata/threads/threads.h" +# include "../../libnetdata/locks/locks.h" +# include "../../libnetdata/avl/avl.h" +# include "../../libnetdata/clocks/clocks.h" +# include "../../libnetdata/config/appconfig.h" +# include "../../libnetdata/ebpf/ebpf.h" + +typedef enum { + MODE_RETURN = 0, //This attaches kprobe when the function returns + MODE_DEVMODE, //This stores log given description about the errors raised + MODE_ENTRY //This attaches kprobe when the function is called +} netdata_run_mode_t; + +typedef struct netdata_syscall_stat { + unsigned long bytes; //total number of bytes + uint64_t call; //total number of calls + uint64_t ecall; //number of calls that returned error + struct netdata_syscall_stat *next; //Link list +}netdata_syscall_stat_t; + +typedef uint64_t netdata_idx_t; + +typedef struct netdata_publish_syscall { + char *dimension; + char *name; + unsigned long nbyte; + unsigned long pbyte; + uint64_t ncall; + uint64_t pcall; + uint64_t nerr; + uint64_t perr; + struct netdata_publish_syscall *next; +}netdata_publish_syscall_t; + +typedef struct netdata_publish_vfs_common { + long write; + long read; + + long running; + long zombie; +}netdata_publish_vfs_common_t; + +typedef struct netdata_error_report { + char comm[16]; + __u32 pid; + + int type; + int err; +}netdata_error_report_t; + +//Chart defintions +# define NETDATA_EBPF_FAMILY "ebpf" +# define NETDATA_FILE_GROUP "File" +# define NETDATA_VFS_GROUP "VFS" +# define NETDATA_PROCESS_GROUP "Process" + +# define NETDATA_FILE_OPEN_CLOSE_COUNT "file_descriptor" +# define NETDATA_FILE_OPEN_ERR_COUNT "file_error" +# define NETDATA_VFS_FILE_CLEAN_COUNT "deleted_objects" +# define NETDATA_VFS_FILE_IO_COUNT "io" +# define NETDATA_VFS_FILE_ERR_COUNT "io_error" + +# define NETDATA_EXIT_SYSCALL "exit" +# define NETDATA_PROCESS_SYSCALL "process_thread" +# define NETDATA_PROCESS_ERROR_NAME "task_error" +# define NETDATA_PROCESS_STATUS_NAME "process_status" + +# define NETDATA_VFS_IO_FILE_BYTES "io_bytes" +# define NETDATA_VFS_DIM_IN_FILE_BYTES "write" +# define NETDATA_VFS_DIM_OUT_FILE_BYTES "read" + +//Log file +# define NETDATA_DEVELOPER_LOG_FILE "developer.log" + +//Maximum number of processors monitored on perf events +# define NETDATA_MAX_PROCESSOR 512 + +//Kernel versions calculated with the formula: +// R = MAJOR*65536 + MINOR*256 + PATCH +# define NETDATA_KERNEL_V5_3 328448 +# define NETDATA_KERNEL_V4_15 265984 + +//Index from kernel +# define NETDATA_KEY_CALLS_DO_SYS_OPEN 0 +# define NETDATA_KEY_ERROR_DO_SYS_OPEN 1 + +# define NETDATA_KEY_CALLS_VFS_WRITE 2 +# define NETDATA_KEY_ERROR_VFS_WRITE 3 +# define NETDATA_KEY_BYTES_VFS_WRITE 4 + +# define NETDATA_KEY_CALLS_VFS_READ 5 +# define NETDATA_KEY_ERROR_VFS_READ 6 +# define NETDATA_KEY_BYTES_VFS_READ 7 + +# define NETDATA_KEY_CALLS_VFS_UNLINK 8 +# define NETDATA_KEY_ERROR_VFS_UNLINK 9 + +# define NETDATA_KEY_CALLS_DO_EXIT 10 + +# define NETDATA_KEY_CALLS_RELEASE_TASK 11 + +# define NETDATA_KEY_CALLS_DO_FORK 12 +# define NETDATA_KEY_ERROR_DO_FORK 13 + +# define NETDATA_KEY_CALLS_CLOSE_FD 14 +# define NETDATA_KEY_ERROR_CLOSE_FD 15 + +# define NETDATA_KEY_CALLS_SYS_CLONE 16 +# define NETDATA_KEY_ERROR_SYS_CLONE 17 + +# define NETDATA_KEY_CALLS_VFS_WRITEV 18 +# define NETDATA_KEY_ERROR_VFS_WRITEV 19 +# define NETDATA_KEY_BYTES_VFS_WRITEV 20 + +# define NETDATA_KEY_CALLS_VFS_READV 21 +# define NETDATA_KEY_ERROR_VFS_READV 22 +# define NETDATA_KEY_BYTES_VFS_READV 23 + +#endif diff --git a/collectors/ebpf_process.plugin/Makefile.am b/collectors/ebpf_process.plugin/Makefile.am deleted file mode 100644 index efb1a97094..0000000000 --- a/collectors/ebpf_process.plugin/Makefile.am +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) - -dist_libconfig_DATA = \ - ebpf_process.conf \ - $(NULL) \ No newline at end of file diff --git a/collectors/ebpf_process.plugin/README.md b/collectors/ebpf_process.plugin/README.md deleted file mode 100644 index cadac9344d..0000000000 --- a/collectors/ebpf_process.plugin/README.md +++ /dev/null @@ -1,220 +0,0 @@ - - -# eBPF monitoring with Netdata - -This collector plugin uses eBPF (Extended Berkeley Packet Filter) to monitor system calls inside your operating system's -kernel. For now, the main goal of this plugin is to monitor IO and process management on the host where it is running. - -
- An example of VFS charts, made possible by the eBPF collector plugin -
An example of VFS charts, made possible by the eBPF collector plugin
-
- -With this eBPF collector, you can monitor sophisticated system-level metrics about your complex applications while -maintaining Netdata's [high standards for performance](#performance). - -## Enable the collector on Linux - -eBPF is only available on Linux systems, which means this collector only works on Linux. - -The collector is currently in an _alpha_ stage, as we are still working on improving compatibility with more Linux -distributions and versions, and to ensure the collector works as expected. - -Follow the next few steps to ensure compatibility, prepare your system, install Netdata with eBPF compiled, and enable -the collector. - -### Ensure kernel compatibility - -To enable this plugin and its collector, you must be on a Linux system with a kernel that is more recent than `4.11.0` -and compiled with the option `CONFIG_KPROBES=y`. You can verify whether your kernel has this option enabled by running -the following commands: - -```bash -grep CONFIG_KPROBES=y /boot/config-$(uname -r) -zgrep CONFIG_KPROBES=y /proc/config.gz -``` - -If `Kprobes` is enabled, you will see `CONFIG_KPROBES=y` as the command's output, and can skip ahead to the next step: [mount `debugfs` and `tracefs`](#mount-debugfs-and-tracefs). - -If you don't see `CONFIG_KPROBES=y` for any of the commands above, you will have to recompile your kernel to enable it. - -The process of recompiling Linux kernels varies based on your distribution and version. Read the documentation for your -system's distribution to learn more about the specific workflow for recompiling the kernel, ensuring that you set the -`CONFIG_KPROBES` setting to `y` in the process. - -- [Ubuntu](https://wiki.ubuntu.com/Kernel/BuildYourOwnKernel) -- [Debian](https://kernel-team.pages.debian.net/kernel-handbook/ch-common-tasks.html#s-common-official) -- [Fedora](https://fedoraproject.org/wiki/Building_a_custom_kernel) -- [CentOS](https://wiki.centos.org/HowTos/Custom_Kernel) -- [Arch Linux](https://wiki.archlinux.org/index.php/Kernel/Traditional_compilation) -- [Slackware](https://docs.slackware.com/howtos:slackware_admin:kernelbuilding) - -### Mount `debugfs` and `tracefs` - -The eBPF collector also requires both the `tracefs` and `debugfs` filesystems. Try mounting the `tracefs` and `debugfs` -filesystems using the commands below: - -```bash -sudo mount -t debugfs nodev /sys/kernel/debug -sudo mount -t tracefs nodev /sys/kernel/tracing -``` -​ -If they are already mounted, you will see an error. If they are not mounted, they should be after running those two -commands. You can also configure your system's `/etc/fstab` configuration to mount these filesystems. - -### Install Netdata with the `--enable-ebpf` - -eBPF collection is only enabled if you install Netdata with the `--enable-ebpf` option. - -If you installed via the [one-line installation script](/packaging/installer/README.md), [64-bit -binary](/packaging/installer/methods/kickstart-64.md), or [manually](/packaging/installer/methods/manual.md), you can -append the `--enable-ebpf` option when you reinstall. - -For example, if you used the one-line installation script, you can reinstall Netdata with the following: - -```bash -bash <(curl -Ss https://my-netdata.io/kickstart.sh) --enable-ebpf -``` - -This process will not overwrite any changes you made to configuration files. - -### Edit `netdata.conf` to enable the collector - -After installing Netdata with the `--enable-ebpf` option, you still need to enable the plugin explicitly. To do so, use -`edit-config` to open `netdata.conf` and set `ebpf_process = yes` in the `[plugins]` section. - -```bash -cd /etc/netdata/ # Replace with your Netdata configuration directory, if not /etc/netdata/ -./edit-config netdata.conf -``` - -Scroll down to the `[plugins]` section and uncomment the `ebpf_process` line after changing its setting to `yes`. - -```conf -[plugins] - ebpf_process = yes -``` - -Restart Netdata with `service netdata restart`, or the appropriate method for your system, and reload your browser to -see eBPF charts. - -## Charts - -The first version of `ebpf_process.plugin` gives a general vision about process running on computer. The charts related -to this plugin are inside the **eBPF** option on dashboard menu and divided in three groups `file`, `vfs`, and -`process`. - -All the collector charts show values per second. The collector retains the total value, but charts only show the -difference between the previous and current metrics collections. - -### File - -This group has two charts to demonstrate how software interacts with the Linux kernel to open and close file -descriptors. - -#### File descriptor - -This chart contains two dimensions that show the number of calls to the functions `do_sys_open` and `__close_fd`. These -functions are not commonly called from software, but they are behind the system cals `open(2)`, `openat(2)`, and -`close(2)`. ​ - -#### File error - -This charts demonstrate the number of times some software tried and failed to open or close a file descriptor. - -### VFS - -A [virtual file system](https://en.wikipedia.org/wiki/Virtual_file_system) (VFS) is a layer on top of regular -filesystems. The functions present inside this API are used for all filesystems, so it's possible the charts in this -group won't show _all_ the actions that occured on your system. - -#### Deleted objects - -This chart monitors calls for `vfs_unlink`. This function is responsible for removing object from the file system. - -#### IO - -This chart shows the number of calls to the functions `vfs_read` and `vfs_write`. - -#### IO bytes - -This chart also monitors `vfs_read` and `vfs_write`, but instead shows the total of bytes read and written with these -functions. - -Netdata displays the number of bytes written as negative, because they are moving down to disk. - -#### IO errors - -Netdata counts and shows the number of instances where a running program experiences a read or write error. - -### Process - -For this group, the eBPF collector monitors process/thread creation and process end, and then displays any errors in the -following charts. - -#### Process thread - -Internally, the Linux kernel treats both process and threads as `tasks`. To create a thread, the kernel offers a few -system calls: `fork(2)`, `vfork(2)` and `clone(2)`. Each of these system calls in turn use the function `_do_fork`. To -generate this chart, Netdata monitors `_do_fork` to populate the `process` dimension, and monitors `sys_clone` to -identify threads - -#### Exit - -Ending a task is actually two steps. The first is a call to the internal function `do_exit`, which notifies the -operating system that the task is finishing its work. The second step is the release of kernel information, which is -done with the internal function `release_task`. The difference between the two dimensions can help you discover [zombie -processes](https://en.wikipedia.org/wiki/Zombie_proces