diff options
author | Markos Fountoulakis <44345837+mfundul@users.noreply.github.com> | 2019-06-21 12:52:58 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-21 12:52:58 +0300 |
commit | 6312080b697dd3b3415a504dd59c351815155e47 (patch) | |
tree | 4af13a96ffd5f303beac2e536366db13fdb6bde0 /daemon | |
parent | 61e45b55a2923a7ff2d2c5fcb07fe5b4e3b43dbc (diff) |
Handle file descriptors running out (#6303)
* Handle file descriptors running out
* Added alarm for dbengine FS and I/O errors
* more verbose alarm message
* * Added File-Descriptor budget to Database Engine instances.
* Changed FD budget of the web server from 50% to 25%.
* Allocated 25% of FDs to dbengine.
* Created a new dbengine global FD utilization chart.
Diffstat (limited to 'daemon')
-rw-r--r-- | daemon/global_statistics.c | 71 |
1 files changed, 70 insertions, 1 deletions
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index fef8eaae6e..53b7546f26 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -538,7 +538,7 @@ void global_statistics_charts(void) { unsigned long long stats_array[RRDENG_NR_STATS]; /* get localhost's DB engine's statistics */ - rrdeng_get_28_statistics(localhost->rrdeng_ctx, stats_array); + rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array); // ---------------------------------------------------------------- @@ -749,6 +749,75 @@ void global_statistics_charts(void) { rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]); rrdset_done(st_io_stats); } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_errors = NULL; + static RRDDIM *rd_fs_errors = NULL; + static RRDDIM *rd_io_errors = NULL; + + if (unlikely(!st_errors)) { + st_errors = rrdset_create_localhost( + "netdata" + , "dbengine_global_errors" + , NULL + , "dbengine" + , NULL + , "NetData DB engine errors" + , "errors/s" + , "netdata" + , "stats" + , 130507 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_errors); + + rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]); + rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]); + rrdset_done(st_errors); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_fd = NULL; + static RRDDIM *rd_fd_current = NULL; + static RRDDIM *rd_fd_max = NULL; + + if (unlikely(!st_fd)) { + st_fd = rrdset_create_localhost( + "netdata" + , "dbengine_global_file_descriptors" + , NULL + , "dbengine" + , NULL + , "NetData DB engine File Descriptors" + , "descriptors" + , "netdata" + , "stats" + , 130508 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_fd_max = rrddim_add(st_fd, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_fd); + + rrddim_set_by_pointer(st_fd, rd_fd_current, (collected_number)stats_array[32]); + /* Careful here, modify this accordingly if the File-Descriptor budget ever changes */ + rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4); + rrdset_done(st_fd); + } } #endif |