tcp syn and accept queue charts and alarms (#4281)

* tcp syn and accept queue charts and alarms; fixes #3234

* tcp syn and accept queue converted to auto

* updated configs.signatures

* enable 1m_ipv4_tcp_accept_queue_drops alarm

* /proc/net/netstat refers to the whole networking stack

* updated configs.signatures

1 files changed, 62 insertions, 8 deletions

diff --git a/conf.d/health.d/tcp_listen.conf b/conf.d/health.d/tcp_listen.conf
index 1bbb8eed8f..552930ab7c 100644
--- a/conf.d/health.d/tcp_listen.conf
+++ b/conf.d/health.d/tcp_listen.conf
@@ -1,28 +1,82 @@
+#
+# There are two queues involved when incoming TCP connections are handled
+# (both at the kernel):
+#
+# SYN queue
+# The SYN queue tracks TCP handshakes until connections are fully established.
+# It overflows when too many incoming TCP connection requests hang in the
+# half-open state and the server is not configured to fall back to SYN cookies.
+# Overflows are usually caused by SYN flood DoS attacks (i.e. someone sends
+# lots of SYN packets and never completes the handshakes).
+#
+# Accept queue
+# The accept queue holds fully established TCP connections waiting to be handled
+# by the listening application. It overflows when the server application fails
+# to accept new connections at the rate they are coming in.
+#
+#
 # -----------------------------------------------------------------------------
-# tcp listen sockets issues
+# tcp accept queue (at the kernel)
 
-   alarm: 1m_ipv4_tcp_listen_overflows
-      on: ipv4.tcplistenissues
-      os: linux freebsd
+   alarm: 1m_tcp_accept_queue_overflows
+      on: ip.tcp_accept_queue
+      os: linux
    hosts: *
   lookup: sum -60s unaligned absolute of ListenOverflows
    units: overflows
    every: 10s
     crit: $this > 0
    delay: up 0 down 5m multiplier 1.5 max 1h
-    info: the number of TCP listen socket overflows during the last minute
+    info: the number of times the TCP accept queue of the kernel overflown, during the last minute
       to: sysadmin
 
-   alarm: 1m_ipv4_tcp_listen_drops
-      on: ipv4.tcplistenissues
+# THIS IS TOO GENERIC
+# CHECK: https://github.com/netdata/netdata/issues/3234#issuecomment-423935842
+   alarm: 1m_tcp_accept_queue_drops
+      on: ip.tcp_accept_queue
       os: linux
    hosts: *
   lookup: sum -60s unaligned absolute of ListenDrops
    units: drops
    every: 10s
+#    warn: $this > 0
+    crit: $this > (($status == $CRITICAL) ? (0) : (150))
+   delay: up 0 down 5m multiplier 1.5 max 1h
+    info: the number of times the TCP accept queue of the kernel dropped packets, during the last minute (includes bogus packets received)
+      to: sysadmin
+
+
+# -----------------------------------------------------------------------------
+# tcp SYN queue (at the kernel)
+
+# When the SYN queue is full, either TcpExtTCPReqQFullDoCookies or
+# TcpExtTCPReqQFullDrop is incremented, depending on whether SYN cookies are
+# enabled or not. In both cases this probably indicates a SYN flood attack,
+# so i guess a notification should be sent.
+
+   alarm: 1m_tcp_syn_queue_drops
+      on: ip.tcp_syn_queue
+      os: linux
+   hosts: *
+  lookup: sum -60s unaligned absolute of TCPReqQFullDrop
+   units: drops
+   every: 10s
+    warn: $this > 0
+    crit: $this > (($status == $CRITICAL) ? (0) : (60))
+   delay: up 0 down 5m multiplier 1.5 max 1h
+    info: the number of times the TCP SYN queue of the kernel was full and dropped packets, during the last minute
+      to: sysadmin
+
+   alarm: 1m_tcp_syn_queue_cookies
+      on: ip.tcp_syn_queue
+      os: linux
+   hosts: *
+  lookup: sum -60s unaligned absolute of TCPReqQFullDoCookies
+   units: cookies
+   every: 10s
     warn: $this > 0
     crit: $this > (($status == $CRITICAL) ? (0) : (60))
    delay: up 0 down 5m multiplier 1.5 max 1h
-    info: the number of TCP listen socket drops during the last minute (includes bogus packets received)
+    info: the number of times the TCP SYN queue of the kernel was full and sent SYN cookies, during the last minute
       to: sysadmin