web_log: add alarm on unmatched lines (#4757)

1 files changed, 30 insertions, 0 deletions

diff --git a/health/health.d/web_log.conf b/health/health.d/web_log.conf
index d8be88b47f..031adc2eac 100644
--- a/health/health.d/web_log.conf
+++ b/health/health.d/web_log.conf
@@ -85,6 +85,36 @@ families: *
     info: the ratio of HTTP internal server errors (5xx), over the last minute
       to: webmaster
 
+# unmatched lines
+
+# the following alarms trigger only when there are enough data.
+# we assume there are enough data when:
+#
+#  $1m_total_requests > 120
+#
+# i.e. when there are at least 120 requests during the last minute
+
+template: 1m_total_requests
+      on: web_log.response_codes
+families: *
+  lookup: sum -1m unaligned
+    calc: ($this == 0)?(1):($this)
+   units: requests
+   every: 10s
+    info: the sum of all HTTP requests over the last minute
+
+template: 1m_unmatched
+on: web_log.response_codes
+families: *
+  lookup: sum -1m unaligned of unmatched
+    calc: $this * 100 / $1m_total_requests
+   units: %
+   every: 10s
+    warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 )
+    crit: ($1m_total_requests > 120) ? ($this > 5) : ( 0 )
+   delay: up 1m down 5m multiplier 1.5 max 1h
+    info: the ratio of unmatched lines, over the last minute
+      to: webmaster
 
 # -----------------------------------------------------------------------------
 # web slow