Add few alarms for elasticsearch (#14197)

Fixes https://github.com/netdata/netdata/issues/14184

2 files changed, 74 insertions, 0 deletions

diff --git a/health/Makefile.am b/health/Makefile.am
index 044ea90766..f0cbb77152 100644
--- a/health/Makefile.am
+++ b/health/Makefile.am
@@ -41,6 +41,7 @@ dist_healthconfig_DATA = \
     health.d/dnsmasq_dhcp.conf \
     health.d/dns_query.conf \
     health.d/dockerd.conf \
+    health.d/elasticsearch.conf \
     health.d/entropy.conf \
     health.d/exporting.conf \
     health.d/geth.conf \
diff --git a/health/health.d/elasticsearch.conf b/health/health.d/elasticsearch.conf
new file mode 100644
index 0000000000..47f8e1eb90
--- /dev/null
+++ b/health/health.d/elasticsearch.conf
@@ -0,0 +1,73 @@
+# you can disable an alarm notification by setting the 'to' line to: silent
+
+# 'red' is a threshold, can't lookup the 'red' dimension - using simple pattern is a workaround.
+
+ template: elasticsearch_cluster_health_status_red
+       on: elasticsearch.cluster_health_status
+    class: Errors
+     type: SearchEngine
+component: Elasticsearch
+   lookup: average -5s unaligned of *ed
+    every: 10s
+    units: status
+     warn: $this == 1
+    delay: down 5m multiplier 1.5 max 1h
+     info: cluster health status is red.
+       to: sysadmin
+
+# the idea of '-10m' is to handle yellow status after node restart,
+# (usually) no action is required because Elasticsearch will automatically restore the green status.
+ template: elasticsearch_cluster_health_status_yellow
+       on: elasticsearch.cluster_health_status
+    class: Errors
+     type: SearchEngine
+component: Elasticsearch
+   lookup: average -10m unaligned of yellow
+    every: 1m
+    units: status
+     warn: $this == 1
+    delay: down 5m multiplier 1.5 max 1h
+     info: cluster health status is yellow.
+       to: sysadmin
+
+ template: elasticsearch_node_index_health_red
+       on: elasticsearch.node_index_health
+    class: Errors
+     type: SearchEngine
+component: Elasticsearch
+   lookup: average -5s unaligned of *ed
+    every: 10s
+    units: status
+     warn: $this == 1
+    delay: down 5m multiplier 1.5 max 1h
+     info: node index $label:index health status is red.
+       to: sysadmin
+
+# don't convert 'lookup' value to seconds in 'calc' due to UI showing seconds as hh:mm:ss (0 as now).
+
+ template: elasticsearch_node_indices_search_time_query
+       on: elasticsearch.node_indices_search_time
+    class: Workload
+     type: SearchEngine
+component: Elasticsearch
+   lookup: average -10m unaligned of query
+    every: 10s
+    units: milliseconds
+     warn: $this > (($status >= $WARNING)  ? (20 * 1000) : (30 * 1000))
+    delay: down 5m multiplier 1.5 max 1h
+     info: search performance is degraded, queries run slowly.
+       to: sysadmin
+
+ template: elasticsearch_node_indices_search_time_fetch
+       on: elasticsearch.node_indices_search_time
+    class: Workload
+     type: SearchEngine
+component: Elasticsearch
+   lookup: average -10m unaligned of fetch
+    every: 10s
+    units: milliseconds
+     warn: $this > (($status >= $WARNING)  ? (3 * 1000) : (5 * 1000))
+     crit: $this > (($status == $CRITICAL) ? (5 * 1000) : (30 * 1000))
+    delay: down 5m multiplier 1.5 max 1h
+     info: search performance is degraded, fetches run slowly.
+       to: sysadmin