go.d: sd local-listeners: discover /proc/net/tcp6 only apps (#17252)

4 files changed, 78 insertions, 68 deletions

diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go
index 8a154cdadf..3ccb12731e 100644
--- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go
+++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go
@@ -13,6 +13,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -183,8 +185,12 @@ func (d *Discoverer) processTargets(tgts []model.Target) []model.TargetGroup {
 }
 
 func (d *Discoverer) parseLocalListeners(bs []byte) ([]model.Target, error) {
-	var tgts []model.Target
-	set := make(map[string]bool)
+	const (
+		local4 = "127.0.0.1"
+		local6 = "::1"
+	)
+
+	var targets []target
 	sc := bufio.NewScanner(bytes.NewReader(bs))
 
 	for sc.Scan() {
@@ -207,11 +213,6 @@ func (d *Discoverer) parseLocalListeners(bs []byte) ([]model.Target, error) {
 			Cmdline:   parts[3],
 		}
 
-		const (
-			local4 = "127.0.0.1"
-			local6 = "::1"
-		)
-
 		if tgt.IPAddress == "0.0.0.0" || strings.HasPrefix(tgt.IPAddress, "127") {
 			tgt.IPAddress = local4
 		} else if tgt.IPAddress == "::" {
@@ -220,21 +221,6 @@ func (d *Discoverer) parseLocalListeners(bs []byte) ([]model.Target, error) {
 
 		tgt.Address = net.JoinHostPort(tgt.IPAddress, tgt.Port)
 
-		key := fmt.Sprintf("%s:%s", tgt.Protocol, tgt.Address)
-		var keyLocal string
-		if strings.HasSuffix(tgt.Protocol, "6") {
-			keyLocal = fmt.Sprintf("%s:%s", tgt.Protocol, net.JoinHostPort(local6, tgt.Port))
-		} else {
-			keyLocal = fmt.Sprintf("%s:%s", tgt.Protocol, net.JoinHostPort(local4, tgt.Port))
-		}
-
-		// Filter targets that accept conns on any (0.0.0.0) and additionally on each individual network interface  (a.b.c.d).
-		// Create a target only for localhost. Assumption: any address always goes first.
-		if set[key] || set[keyLocal] {
-			continue
-		}
-		set[key] = true
-
 		hash, err := calcHash(tgt)
 		if err != nil {
 			continue
@@ -243,10 +229,49 @@ func (d *Discoverer) parseLocalListeners(bs []byte) ([]model.Target, error) {
 		tgt.hash = hash
 		tgt.Tags().Merge(d.Tags())
 
-		tgts = append(tgts, &tgt)
+		targets = append(targets, tgt)
+	}
+
+	// order: TCP, TCP6, UDP, UDP6
+	sort.Slice(targets, func(i, j int) bool {
+		tgt1, tgt2 := targets[i], targets[j]
+		if tgt1.Protocol != tgt2.Protocol {
+			return tgt1.Protocol < tgt2.Protocol
+		}
+
+		p1, _ := strconv.Atoi(targets[i].Port)
+		p2, _ := strconv.Atoi(targets[j].Port)
+		if p1 != p2 {
+			return p1 < p2
+		}
+
+		return tgt1.IPAddress == local4 || tgt1.IPAddress == local6
+	})
+
+	seen := make(map[string]bool)
+	tgts := make([]model.Target, len(targets))
+	var n int
+
+	for _, tgt := range targets {
+		tgt := tgt
+
+		proto := strings.TrimSuffix(tgt.Protocol, "6")
+		key := tgt.Protocol + ":" + tgt.Address
+		keyLocal4 := proto + ":" + net.JoinHostPort(local4, tgt.Port)
+		keyLocal6 := proto + "6:" + net.JoinHostPort(local6, tgt.Port)
+
+		// Filter targets that accept conns on any (0.0.0.0) and additionally on each individual network interface  (a.b.c.d).
+		// Create a target only for localhost. Assumption: any address always goes first.
+		if seen[key] || seen[keyLocal4] || seen[keyLocal6] {
+			continue
+		}
+		seen[key] = true
+
+		tgts[n] = &tgt
+		n++
 	}
 
-	return tgts, nil
+	return tgts[:n], nil
 }
 
 type localListenersExec struct {
@@ -258,11 +283,10 @@ func (e *localListenersExec) discover(ctx context.Context) ([]byte, error) {
 	execCtx, cancel := context.WithTimeout(ctx, e.timeout)
 	defer cancel()
 
-	// TCPv4 and UPDv4 sockets in LISTEN state
+	// TCPv4/6 and UPDv4 sockets in LISTEN state
 	// https://github.com/netdata/netdata/blob/master/src/collectors/plugins.d/local_listeners.c
 	args := []string{
 		"no-udp6",
-		"no-tcp6",
 		"no-local",
 		"no-inbound",
 		"no-outbound",
diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go
index e81dcc57a9..c49e816ac9 100644
--- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go
+++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go
@@ -21,6 +21,8 @@ func TestDiscoverer_Discover(t *testing.T) {
 				cli.addListener("TCP|0.0.0.0|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D")
 				cli.addListener("TCP|192.0.2.1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D")
 				cli.addListener("UDP|127.0.0.1|53768|/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1")
+				cli.addListener("TCP6|::|80|/usr/sbin/apache2 -k start")
+				cli.addListener("TCP|0.0.0.0|80|/usr/sbin/apache2 -k start")
 				time.Sleep(interval * 2)
 			},
 			wantGroups: []model.TargetGroup{&targetGroup{
@@ -28,20 +30,12 @@ func TestDiscoverer_Discover(t *testing.T) {
 				source:   "discoverer=net_listeners,host=localhost",
 				targets: []model.Target{
 					withHash(&target{
-						Protocol:  "UDP6",
-						IPAddress: "::1",
-						Port:      "8125",
-						Address:   "[::1]:8125",
-						Comm:      "netdata",
-						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
-					}),
-					withHash(&target{
-						Protocol:  "TCP6",
-						IPAddress: "::1",
-						Port:      "8125",
-						Address:   "[::1]:8125",
-						Comm:      "netdata",
-						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
+						Protocol:  "TCP",
+						IPAddress: "127.0.0.1",
+						Port:      "80",
+						Address:   "127.0.0.1:80",
+						Comm:      "apache2",
+						Cmdline:   "/usr/sbin/apache2 -k start",
 					}),
 					withHash(&target{
 						Protocol:  "TCP",
@@ -59,6 +53,14 @@ func TestDiscoverer_Discover(t *testing.T) {
 						Comm:      "go.d.plugin",
 						Cmdline:   "/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1",
 					}),
+					withHash(&target{
+						Protocol:  "UDP6",
+						IPAddress: "::1",
+						Port:      "8125",
+						Address:   "[::1]:8125",
+						Comm:      "netdata",
+						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
+					}),
 				},
 			}},
 		},
@@ -86,14 +88,6 @@ func TestDiscoverer_Discover(t *testing.T) {
 						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
 					}),
 					withHash(&target{
-						Protocol:  "TCP6",
-						IPAddress: "::1",
-						Port:      "8125",
-						Address:   "[::1]:8125",
-						Comm:      "netdata",
-						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
-					}),
-					withHash(&target{
 						Protocol:  "TCP",
 						IPAddress: "127.0.0.1",
 						Port:      "8125",
@@ -128,14 +122,6 @@ func TestDiscoverer_Discover(t *testing.T) {
 				source:   "discoverer=net_listeners,host=localhost",
 				targets: []model.Target{
 					withHash(&target{
-						Protocol:  "TCP6",
-						IPAddress: "::1",
-						Port:      "8125",
-						Address:   "[::1]:8125",
-						Comm:      "netdata",
-						Cmdline:   "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D",
-					}),
-					withHash(&target{
 						Protocol:  "TCP",
 						IPAddress: "127.0.0.1",
 						Port:      "8125",
diff --git a/src/go/collectors/go.d.plugin/config/go.d/sd/docker.conf b/src/go/collectors/go.d.plugin/config/go.d/sd/docker.conf
index c63cfa55a6..6fd4004132 100644
--- a/src/go/collectors/go.d.plugin/config/go.d/sd/docker.conf
+++ b/src/go/collectors/go.d.plugin/config/go.d/sd/docker.conf
@@ -129,7 +129,7 @@ compose:
         template: |
           module: mysql
           name: docker_{{.Name}}
-          dsn: netdata@tcp({{.IPAddress}}:{{.PrivatePort}})/
+          dsn: netdata@tcp({{.Address}})/
       - selector: "nginx"
         template: |
           module: nginx
@@ -139,22 +139,22 @@ compose:
         template: |
           module: pgbouncer
           name: docker_{{.Name}}
-          dsn: postgres://netdata:postgres@{{.IPAddress}}:{{.PrivatePort}}/pgbouncer
+          dsn: postgres://netdata:postgres@{{.Address}}/pgbouncer
       - selector: "pika"
         template: |
           module: pika
           name: docker_{{.Name}}
-          address: redis://@{{.IPAddress}}:{{.PrivatePort}}
+          address: redis://@{{.Address}}
       - selector: "postgres"
         template: |
           module: postgres
           name: docker_{{.Name}}
-          dsn: postgres://netdata:postgres@{{.IPAddress}}:{{.PrivatePort}}/postgres
+          dsn: postgres://netdata:postgres@{{.Address}}/postgres
       - selector: "proxysql"
         template: |
           module: proxysql
           name: docker_{{.Name}}
-          dsn: stats:stats@tcp({{.IPAddress}}:{{.PrivatePort}})/
+          dsn: stats:stats@tcp({{.Address}})/
       - selector: "rabbitmq"
         template: |
           module: rabbitmq
@@ -164,7 +164,7 @@ compose:
         template: |
           module: redis
           name: docker_{{.Name}}
-          address: redis://@{{.IPAddress}}:{{.PrivatePort}}
+          address: redis://@{{.Address}}
       - selector: "tengine"
         template: |
           module: tengine
diff --git a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf
index 612779bc4a..be5b302890 100644
--- a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf
+++ b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf
@@ -15,7 +15,7 @@ classify:
       - tags: "activemq"
         expr: '{{ and (eq .Port "8161") (eq .Comm "activemq") }}'
       - tags: "apache"
-        expr: '{{ and (eq .Port "80" "8080") (eq .Comm "apache" "httpd") }}'
+        expr: '{{ and (eq .Port "80" "8080") (eq .Comm "apache" "apache2" "httpd") }}'
       - tags: "bind"
         expr: '{{ and (eq .Port "8653") (eq .Comm "bind" "named") }}'
       - tags: "cassandra"
@@ -270,7 +270,7 @@ compose:
         template: |
           module: mysql
           name: local
-          dsn: netdata@tcp({{.IPAddress}}:{{.Port}})/
+          dsn: netdata@tcp({{.Address}})/
       - selector: "nginx"
         template: |
           - module: nginx
@@ -300,7 +300,7 @@ compose:
         template: |
           module: pgbouncer
           name: local
-          dsn: postgres://netdata:postgres@{{.IPAddress}}:{{.Port}}/pgbouncer
+          dsn: postgres://netdata:postgres@{{.Address}}/pgbouncer
       - selector: "pihole"
         template: |
           module: pihole
@@ -315,7 +315,7 @@ compose:
         template: |
           module: postgres
           name: local
-          dsn: postgresql://netdata@{{.IPAddress}}:{{.Port}}/postgres
+          dsn: postgresql://netdata@{{.Address}}/postgres
       - selector: "powerdns"
         template: |
           module: powerdns
@@ -334,7 +334,7 @@ compose:
         template: |
           module: proxysql
           name: local
-          dsn: stats:stats@tcp({{.IPAddress}}:{{.Port}})/
+          dsn: stats:stats@tcp({{.Address}})/
       - selector: "rabbitmq"
         template: |
           module: rabbitmq
@@ -347,7 +347,7 @@ compose:
         template: |
           module: redis
           name: local
-          address: redis://@{{.IPAddress}}:{{.Port}}
+          address: redis://@{{.Address}}
       - selector: "supervisord"
         template: |
           module: supervisord