diff options
author | Ilya Mashchenko <ilya@netdata.cloud> | 2023-01-06 14:49:32 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-06 14:49:32 +0200 |
commit | bc977526fcc612937f56a0c4e86432a454df3310 (patch) | |
tree | 093ea61ae67789366164b032cbaea962ccecd0bf | |
parent | cd80dd6a04d4d3467189e63c29f01a01b229c2d8 (diff) |
Add few alarms for elasticsearch (#14197)
Fixes https://github.com/netdata/netdata/issues/14184
-rw-r--r-- | health/Makefile.am | 1 | ||||
-rw-r--r-- | health/health.d/elasticsearch.conf | 73 |
2 files changed, 74 insertions, 0 deletions
diff --git a/health/Makefile.am b/health/Makefile.am index 044ea90766..f0cbb77152 100644 --- a/health/Makefile.am +++ b/health/Makefile.am @@ -41,6 +41,7 @@ dist_healthconfig_DATA = \ health.d/dnsmasq_dhcp.conf \ health.d/dns_query.conf \ health.d/dockerd.conf \ + health.d/elasticsearch.conf \ health.d/entropy.conf \ health.d/exporting.conf \ health.d/geth.conf \ diff --git a/health/health.d/elasticsearch.conf b/health/health.d/elasticsearch.conf new file mode 100644 index 0000000000..47f8e1eb90 --- /dev/null +++ b/health/health.d/elasticsearch.conf @@ -0,0 +1,73 @@ +# you can disable an alarm notification by setting the 'to' line to: silent + +# 'red' is a threshold, can't lookup the 'red' dimension - using simple pattern is a workaround. + + template: elasticsearch_cluster_health_status_red + on: elasticsearch.cluster_health_status + class: Errors + type: SearchEngine +component: Elasticsearch + lookup: average -5s unaligned of *ed + every: 10s + units: status + warn: $this == 1 + delay: down 5m multiplier 1.5 max 1h + info: cluster health status is red. + to: sysadmin + +# the idea of '-10m' is to handle yellow status after node restart, +# (usually) no action is required because Elasticsearch will automatically restore the green status. + template: elasticsearch_cluster_health_status_yellow + on: elasticsearch.cluster_health_status + class: Errors + type: SearchEngine +component: Elasticsearch + lookup: average -10m unaligned of yellow + every: 1m + units: status + warn: $this == 1 + delay: down 5m multiplier 1.5 max 1h + info: cluster health status is yellow. + to: sysadmin + + template: elasticsearch_node_index_health_red + on: elasticsearch.node_index_health + class: Errors + type: SearchEngine +component: Elasticsearch + lookup: average -5s unaligned of *ed + every: 10s + units: status + warn: $this == 1 + delay: down 5m multiplier 1.5 max 1h + info: node index $label:index health status is red. + to: sysadmin + +# don't convert 'lookup' value to seconds in 'calc' due to UI showing seconds as hh:mm:ss (0 as now). + + template: elasticsearch_node_indices_search_time_query + on: elasticsearch.node_indices_search_time + class: Workload + type: SearchEngine +component: Elasticsearch + lookup: average -10m unaligned of query + every: 10s + units: milliseconds + warn: $this > (($status >= $WARNING) ? (20 * 1000) : (30 * 1000)) + delay: down 5m multiplier 1.5 max 1h + info: search performance is degraded, queries run slowly. + to: sysadmin + + template: elasticsearch_node_indices_search_time_fetch + on: elasticsearch.node_indices_search_time + class: Workload + type: SearchEngine +component: Elasticsearch + lookup: average -10m unaligned of fetch + every: 10s + units: milliseconds + warn: $this > (($status >= $WARNING) ? (3 * 1000) : (5 * 1000)) + crit: $this > (($status == $CRITICAL) ? (5 * 1000) : (30 * 1000)) + delay: down 5m multiplier 1.5 max 1h + info: search performance is degraded, fetches run slowly. + to: sysadmin |