From 2728be8b0614fd58d4b530a32fee164509e88fd8 Mon Sep 17 00:00:00 2001 From: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com> Date: Tue, 24 Sep 2019 11:59:15 +0300 Subject: Detect deadlock in dbengine page cache (#6911) * Detect deadlock in dbengine page cache when there are too many metrics and print error message * Resolve dbengine deadlock by dropping metrics when page cache is too small and define relevant alarms * Changed printing deadlock errors to only happen once per dbengine instance --- daemon/global_statistics.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'daemon/global_statistics.c') diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 53b7546f26..777ec2816a 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -538,7 +538,7 @@ void global_statistics_charts(void) { unsigned long long stats_array[RRDENG_NR_STATS]; /* get localhost's DB engine's statistics */ - rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array); + rrdeng_get_35_statistics(localhost->rrdeng_ctx, stats_array); // ---------------------------------------------------------------- @@ -756,6 +756,8 @@ void global_statistics_charts(void) { static RRDSET *st_errors = NULL; static RRDDIM *rd_fs_errors = NULL; static RRDDIM *rd_io_errors = NULL; + static RRDDIM *rd_pg_cache_warnings = NULL; + static RRDDIM *rd_pg_cache_errors = NULL; if (unlikely(!st_errors)) { st_errors = rrdset_create_localhost( @@ -775,12 +777,17 @@ void global_statistics_charts(void) { rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_pg_cache_warnings = rrddim_add(st_errors, "Page-Cache warnings", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + rd_pg_cache_errors = rrddim_add(st_errors, "Page-Cache errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st_errors); rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]); rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]); + rrddim_set_by_pointer(st_errors, rd_pg_cache_warnings, (collected_number)stats_array[33]); + rrddim_set_by_pointer(st_errors, rd_pg_cache_errors, (collected_number)stats_array[34]); rrdset_done(st_errors); } -- cgit v1.2.3