From 954f91e504c7660ea4e1e1763afe9e4de89c404a Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Wed, 20 Mar 2024 12:25:04 +0200 Subject: go.d: local-listeners sd: use "ip:port" as address instead of "localhost" (#17203) --- .../sd/discoverer/netlisteners/netlisteners.go | 44 ++++++-- .../discoverer/netlisteners/netlisteners_test.go | 114 ++++++++++++--------- .../sd/discoverer/netlisteners/sim_test.go | 15 +-- .../discovery/sd/discoverer/netlisteners/target.go | 12 ++- .../go.d.plugin/config/go.d/sd/net_listeners.conf | 110 ++++++++++---------- 5 files changed, 171 insertions(+), 124 deletions(-) diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go index 75fa4274b6..8a154cdadf 100644 --- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go +++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "log/slog" + "net" "os" "os/exec" "path/filepath" @@ -183,27 +184,56 @@ func (d *Discoverer) processTargets(tgts []model.Target) []model.TargetGroup { func (d *Discoverer) parseLocalListeners(bs []byte) ([]model.Target, error) { var tgts []model.Target - + set := make(map[string]bool) sc := bufio.NewScanner(bytes.NewReader(bs)) + for sc.Scan() { text := strings.TrimSpace(sc.Text()) if text == "" { continue } - // Protocol|Address|Port|Cmdline + // Protocol|IPAddress|Port|Cmdline parts := strings.SplitN(text, "|", 4) if len(parts) != 4 { return nil, fmt.Errorf("unexpected data: '%s'", text) } tgt := target{ - Protocol: parts[0], - Address: parts[1], - Port: parts[2], - Comm: extractComm(parts[3]), - Cmdline: parts[3], + Protocol: parts[0], + IPAddress: parts[1], + Port: parts[2], + Comm: extractComm(parts[3]), + Cmdline: parts[3], + } + + const ( + local4 = "127.0.0.1" + local6 = "::1" + ) + + if tgt.IPAddress == "0.0.0.0" || strings.HasPrefix(tgt.IPAddress, "127") { + tgt.IPAddress = local4 + } else if tgt.IPAddress == "::" { + tgt.IPAddress = local6 + } + + tgt.Address = net.JoinHostPort(tgt.IPAddress, tgt.Port) + + key := fmt.Sprintf("%s:%s", tgt.Protocol, tgt.Address) + var keyLocal string + if strings.HasSuffix(tgt.Protocol, "6") { + keyLocal = fmt.Sprintf("%s:%s", tgt.Protocol, net.JoinHostPort(local6, tgt.Port)) + } else { + keyLocal = fmt.Sprintf("%s:%s", tgt.Protocol, net.JoinHostPort(local4, tgt.Port)) + } + + // Filter targets that accept conns on any (0.0.0.0) and additionally on each individual network interface (a.b.c.d). + // Create a target only for localhost. Assumption: any address always goes first. + if set[key] || set[keyLocal] { + continue } + set[key] = true hash, err := calcHash(tgt) if err != nil { diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go index b724517a09..e81dcc57a9 100644 --- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go +++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners_test.go @@ -15,7 +15,11 @@ func TestDiscoverer_Discover(t *testing.T) { listenersCli: func(cli listenersCli, interval, expiry time.Duration) { cli.addListener("UDP6|::1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") cli.addListener("TCP6|::1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") + cli.addListener("TCP6|::|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") + cli.addListener("TCP6|2001:DB8::1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") cli.addListener("TCP|127.0.0.1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") + cli.addListener("TCP|0.0.0.0|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") + cli.addListener("TCP|192.0.2.1|8125|/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D") cli.addListener("UDP|127.0.0.1|53768|/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1") time.Sleep(interval * 2) }, @@ -24,32 +28,36 @@ func TestDiscoverer_Discover(t *testing.T) { source: "discoverer=net_listeners,host=localhost", targets: []model.Target{ withHash(&target{ - Protocol: "UDP6", - Address: "::1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "UDP6", + IPAddress: "::1", + Port: "8125", + Address: "[::1]:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "TCP6", - Address: "::1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP6", + IPAddress: "::1", + Port: "8125", + Address: "[::1]:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "TCP", - Address: "127.0.0.1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP", + IPAddress: "127.0.0.1", + Port: "8125", + Address: "127.0.0.1:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "UDP", - Address: "127.0.0.1", - Port: "53768", - Comm: "go.d.plugin", - Cmdline: "/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1", + Protocol: "UDP", + IPAddress: "127.0.0.1", + Port: "53768", + Address: "127.0.0.1:53768", + Comm: "go.d.plugin", + Cmdline: "/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1", }), }, }}, @@ -70,32 +78,36 @@ func TestDiscoverer_Discover(t *testing.T) { source: "discoverer=net_listeners,host=localhost", targets: []model.Target{ withHash(&target{ - Protocol: "UDP6", - Address: "::1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "UDP6", + IPAddress: "::1", + Port: "8125", + Address: "[::1]:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "TCP6", - Address: "::1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP6", + IPAddress: "::1", + Port: "8125", + Address: "[::1]:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "TCP", - Address: "127.0.0.1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP", + IPAddress: "127.0.0.1", + Port: "8125", + Address: "127.0.0.1:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "UDP", - Address: "127.0.0.1", - Port: "53768", - Comm: "go.d.plugin", - Cmdline: "/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1", + Protocol: "UDP", + IPAddress: "127.0.0.1", + Port: "53768", + Address: "127.0.0.1:53768", + Comm: "go.d.plugin", + Cmdline: "/opt/netdata/usr/libexec/netdata/plugins.d/go.d.plugin 1", }), }, }}, @@ -116,18 +128,20 @@ func TestDiscoverer_Discover(t *testing.T) { source: "discoverer=net_listeners,host=localhost", targets: []model.Target{ withHash(&target{ - Protocol: "TCP6", - Address: "::1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP6", + IPAddress: "::1", + Port: "8125", + Address: "[::1]:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), withHash(&target{ - Protocol: "TCP", - Address: "127.0.0.1", - Port: "8125", - Comm: "netdata", - Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", + Protocol: "TCP", + IPAddress: "127.0.0.1", + Port: "8125", + Address: "127.0.0.1:8125", + Comm: "netdata", + Cmdline: "/opt/netdata/usr/sbin/netdata -P /run/netdata/netdata.pid -D", }), }, }}, diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/sim_test.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/sim_test.go index f13d01c691..ad90f8278b 100644 --- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/sim_test.go +++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/sim_test.go @@ -5,6 +5,7 @@ package netlisteners import ( "context" "errors" + "slices" "sort" "strings" "sync" @@ -103,29 +104,29 @@ func (sim *discoverySim) run(t *testing.T) { } func newMockLocalListenersExec() *mockLocalListenersExec { - return &mockLocalListenersExec{ - listeners: make(map[string]bool), - } + return &mockLocalListenersExec{} } type mockLocalListenersExec struct { errResponse bool mux sync.Mutex - listeners map[string]bool + listeners []string } func (m *mockLocalListenersExec) addListener(s string) { m.mux.Lock() defer m.mux.Unlock() - m.listeners[s] = true + m.listeners = append(m.listeners, s) } func (m *mockLocalListenersExec) removeListener(s string) { m.mux.Lock() defer m.mux.Unlock() - delete(m.listeners, s) + if i := slices.Index(m.listeners, s); i != -1 { + m.listeners = append(m.listeners[:i], m.listeners[i+1:]...) + } } func (m *mockLocalListenersExec) discover(context.Context) ([]byte, error) { @@ -137,7 +138,7 @@ func (m *mockLocalListenersExec) discover(context.Context) ([]byte, error) { defer m.mux.Unlock() var buf strings.Builder - for s := range m.listeners { + for _, s := range m.listeners { buf.WriteString(s) buf.WriteByte('\n') } diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/target.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/target.go index 501b280ba1..a36620f32e 100644 --- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/target.go +++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/target.go @@ -24,11 +24,13 @@ type target struct { hash uint64 - Protocol string - Address string - Port string - Comm string - Cmdline string + Protocol string + IPAddress string + Port string + Comm string + Cmdline string + + Address string // "IPAddress:Port" } func (t *target) TUID() string { return tuid(t) } diff --git a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf index dd11977ee9..10e622bd48 100644 --- a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf +++ b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf @@ -35,7 +35,7 @@ classify: - tags: "dnsdist" expr: '{{ and (eq .Port "8083") (eq .Comm "dnsdist") }}' - tags: "dnsmasq" - expr: '{{ and (eq .Port "53") (eq .Comm "dnsmasq") }}' + expr: '{{ and (eq .Protocol "UDP") (eq .Port "53") (eq .Comm "dnsmasq") }}' - tags: "docker_engine" expr: '{{ and (eq .Port "9323") (eq .Comm "dockerd") }}' - tags: "elasticsearch" @@ -111,7 +111,7 @@ classify: tags: "-unknown exporter" match: - tags: "exporter" - expr: '{{ not (empty (promPort .Port)) }}' + expr: '{{ and (not (empty (promPort .Port))) (not (eq .Comm "docker-proxy")) }}' compose: - name: "Applications" selector: "app" @@ -120,59 +120,59 @@ compose: template: | module: activemq name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} webadmin: admin - selector: "apache" template: | module: apache name: local - url: http://localhost:{{.Port}}/server-status?auto + url: http://{{.Address}}/server-status?auto - selector: "bind" template: | module: bind name: local - url: http://localhost:{{.Port}}/json/v1 + url: http://{{.Address}}/json/v1 - selector: "cassandra" template: | module: cassandra name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "chrony" template: | module: chrony name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - selector: "cockroachdb" template: | module: cockroachdb name: local - url: http://localhost:{{.Port}}/_status/vars + url: http://{{.Address}}/_status/vars - selector: "consul" template: | module: consul name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} - selector: "coredns" template: | module: coredns name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "couchbase" template: | module: couchbase name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} - selector: "couchdb" template: | module: couchdb name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} node: '_local' - selector: "dnsdist" template: | module: dnsdist name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} headers: X-API-Key: 'dnsdist-api-key' - selector: "dnsmasq" @@ -180,23 +180,23 @@ compose: module: dnsmasq name: local protocol: udp - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - selector: "docker_engine" template: | module: docker_engine name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "elasticsearch" template: | module: elasticsearch name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} cluster_mode: no - selector: "opensearch" template: | module: elasticsearch name: local - url: https://localhost:{{.Port}} + url: https://{{.Address}} cluster_mode: no tls_skip_verify: yes username: admin @@ -205,148 +205,148 @@ compose: template: | module: envoy name: local - url: http://localhost:{{.Port}}/stats/prometheus + url: http://{{.Address}}/stats/prometheus - selector: "envoy" template: | module: envoy name: local - url: http://localhost:{{.Port}}/stats/prometheus + url: http://{{.Address}}/stats/prometheus - selector: "fluentd" template: | module: fluentd name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} - selector: "freeradius" template: | module: freeradius name: local - address: localhost + address: {{.IPAddress}} port: {{.Port}} secret: adminsecret - selector: "geth" template: | module: geth name: local - url: http://localhost:{{.Port}}/debug/metrics/prometheus + url: http://{{.Address}}/debug/metrics/prometheus - selector: "haproxy" template: | module: haproxy name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "hdfs_namenode" template: | module: hdfs name: namenode_local - url: http://localhost:{{.Port}}/jmx + url: http://{{.Address}}/jmx - selector: "hdfs_datanode" template: | module: hdfs name: datanode_local - url: http://localhost:{{.Port}}/jmx + url: http://{{.Address}}/jmx - selector: "kubelet" template: | module: kubelet name: local {{- if eq .Port "10255" }} - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics {{- else }} - url: https://localhost:{{.Port}}/metrics + url: https://{{.Address}}/metrics tls_skip_verify: yes {{- end }} - selector: "kubeproxy" template: | module: kubeproxy name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "lighttpd" template: | module: lighttpd name: local - url: http://localhost:{{.Port}}/server-status?auto + url: http://{{.Address}}/server-status?auto - selector: "logstash" template: | module: logstash name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} - selector: "mongodb" template: | module: mongodb name: local - uri: mongodb://localhost:{{.Port}} + uri: mongodb://{{.Address}} - selector: "mysql" template: | module: mysql name: local - dsn: netdata@tcp(127.0.0.1:{{.Port}})/ + dsn: netdata@tcp({{.IPAddress}}:{{.Port}})/ - selector: "nginx" template: | - module: nginx name: local - url: http://localhost:{{.Port}}/stub_status + url: http://{{.Address}}/stub_status - module: nginx name: local - url: http://localhost:{{.Port}}/basic_status + url: http://{{.Address}}/basic_status - module: nginx name: local - url: http://localhost:{{.Port}}/nginx_status + url: http://{{.Address}}/nginx_status - module: nginx name: local - url: http://localhost:{{.Port}}/status + url: http://{{.Address}}/status - selector: "ntpd" template: | module: ntpd name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} collect_peers: no - selector: "openvpn" template: | module: openvpn name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - selector: "pgbouncer" template: | module: pgbouncer name: local - dsn: postgres://netdata:postgres@127.0.0.1:{{.Port}}/pgbouncer + dsn: postgres://netdata:postgres@{{.IPAddress}}:{{.Port}}/pgbouncer - selector: "pihole" template: | module: pihole name: local - url: http://localhost + url: http://{{.Address}} - selector: "pika" template: | module: pika name: local - address: redis://@127.0.0.1:{{.Port}} + address: redis://@{{.IPAddress}}:{{.Port}} - selector: "postgres" template: | module: postgres name: local - dsn: postgresql://netdata@127.0.0.1:{{.Port}}/postgres + dsn: postgresql://netdata@{{.IPAddress}}:{{.Port}}/postgres - selector: "powerdns" template: | module: powerdns name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} headers: X-API-KEY: secret - selector: "powerdns_recursor" template: | module: powerdns_recursor name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} headers: X-API-KEY: secret - selector: "proxysql" template: | module: proxysql name: local - dsn: stats:stats@tcp(127.0.0.1:{{.Port}})/ + dsn: stats:stats@tcp({{.IPAddress}}:{{.Port}})/ - selector: "rabbitmq" template: | module: rabbitmq name: local - url: http://localhost:{{.Port}} + url: http://{{.Address}} username: guest password: guest collect_queues_metrics: no @@ -354,42 +354,42 @@ compose: template: | module: redis name: local - address: redis://@127.0.0.1:{{.Port}} + address: redis://@{{.IPAddress}}:{{.Port}} - selector: "supervisord" template: | module: supervisord name: local - url: http://localhost:{{.Port}}/RPC2 + url: http://{{.Address}}/RPC2 - selector: "traefik" template: | module: traefik name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "traefik" template: | module: traefik name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "unbound" template: | module: unbound name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - selector: "upsd" template: | module: upsd name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - selector: "vernemq" template: | module: vernemq name: local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics - selector: "zookeeper" template: | module: zookeeper name: local - address: 127.0.0.1:{{.Port}} + address: {{.Address}} - name: "Prometheus exporters generic" selector: "exporter" @@ -399,7 +399,7 @@ compose: {{ $name := promPort .Port -}} module: prometheus name: {{$name}}_local - url: http://localhost:{{.Port}}/metrics + url: http://{{.Address}}/metrics {{ if eq $name "caddy" -}} expected_prefix: 'caddy_' {{ else if eq $name "openethereum" -}} -- cgit v1.2.3