Code reorg and cleanup - enrichment of /api/v2 (#15294)

* claim script now accepts the same params as the kickstart

* rewrote buildinfo to unify all methods

* added cloud unavailable in cloud status

* added all exporters

* renamed httpd to h2o

* rename ENABLE_COMPRESSION to ENABLE_LZ4

* rename global variable

* rename ENABLE_HTTPS to ENABLE_OPENSSL

* fix coverity-scan for openssl

* add lz4 to coverity-scan

* added all plugins and most of the features

* added all plugins and most of the features

* generalize bitmap code so that we can have any size of bitmaps

* cleanup

* fix compilation without protobuf

* fix compilation with others allocators

* fix bitmap

* comprehensive bitmaps unit test

* bitmap as macros

* added developer mode

* added system info to build info

* cloud available/unavailable

* added /api/v2/info

* added units and ni to transitions

* when showing instances and transitions, show only the instances that have transitions

* cleanup

* add missing quotes

* add anchor to transitions

* added more to build info

* calculate retention per tier and expose it to /api/v2/info

* added currently collected metrics

* do not show space and retention when no numbers are available

* fix impossible overflow

* Add function for transitions and execute callback

* In case of error, reset and try next dictionary entry

* Fix error message

* simpler logic to maintain retention per tier

* /api/v2/alert_transitions

* Handle case of recipient null
Convert after and before to usec

* Add classification, type and component

* working /api/v2/alert_transitions

* Fix query to properly handle context and alert name

* cleanup

* Add search with transition

* accept transition in /api/v2/alert_transitions

* totaly dynamic facets

* fixed debug info

* restructured facets

* cleanup; removal of options=transitions

* updated alert entries flags

* method to exec

* Return also exec run timestamp
Temp table cleanup only when we don't execute with a transition

* cleanup obsolete anchor parameter

* Add sql_get_alert_configuration function

* added options=config to alert_transitions

* added /api/v2/alert_config

* preliminary work for /api/v2/claim

* initialize variables; do not expose expected retention if no disk space info is available; do not report aclk as initializing when not claimed

* fix claim session key filename

* put a newline into the session key file

* more progress on claiming

* final /api/v2/claim endpoint

* after claiming, refresh our state at the output

* Fix query to fetch config

* Remove debug log

* add configuration objects

* add configuration objects - fixed

* respect the NETDATA_DISABLE_CLOUD env variable

* NETDATA_DISABLE_CLOUD env variable sets the default, but the config sets the final value

* use a new claimed_id on every claiming

* regenerate random key on claiming and wait for online status

* ignore write() return value when writing a newline

* dont show cloud status disabled when claimed_id is missing

* added ctx to alert instances

* cleanup config and transitions from /api/v2/alerts

* fix unused variable

* in /api/v2/alert_config show 1 config without an array

* show alert values conditionally, by appending options=values

* When storing host info if the key value is empty, store unknown

* added options=summary to control when the alerts summary is shown

* increased http_api_v2 to version 5

* claming random key file is now not world readable

* added local-listeners binary that detects all the listening ports, their IPs and their command lines

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 112 insertions, 101 deletions

diff --git a/Makefile.am b/Makefile.am
index d73ad972b0..c6f55d4dc9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -84,7 +84,7 @@ dist_noinst_DATA = \
     packaging/protobuf.version \
     packaging/version \
     database/engine/journalfile_v2.ksy.in \
-    httpd/h2o \
+    web/server/h2o/libh2o \
     $(NULL)
 
 # until integrated within build
@@ -281,6 +281,10 @@ CGROUP_NETWORK_FILES = \
     $(LIBNETDATA_FILES) \
     $(NULL)
 
+LOCAL_LISTENERS_FILES = \
+    collectors/plugins.d/local_listeners.c \
+    $(NULL)
+
 DISKSPACE_PLUGIN_FILES = \
     collectors/diskspace.plugin/plugin_diskspace.c \
     $(NULL)
@@ -945,111 +949,111 @@ DAEMON_FILES = \
     daemon/unit_test.h \
     $(NULL)
 
-HTTPD_FILES = \
-    httpd/http_server.c \
-    httpd/http_server.h \
-    httpd/h2o_utils.c \
-    httpd/h2o_utils.h \
+H2O_FILES = \
+    web/server/h2o/http_server.c \
+    web/server/h2o/http_server.h \
+    web/server/h2o/h2o_utils.c \
+    web/server/h2o/h2o_utils.h \
     $(NULL)
 
 libh2o_a_SOURCES = \
-    httpd/h2o/deps/cloexec/cloexec.c \
-    httpd/h2o/deps/libgkc/gkc.c \
-    httpd/h2o/deps/libyrmcds/close.c \
-    httpd/h2o/deps/libyrmcds/connect.c \
-    httpd/h2o/deps/libyrmcds/recv.c \
-    httpd/h2o/deps/libyrmcds/send.c \
-    httpd/h2o/deps/libyrmcds/send_text.c \
-    httpd/h2o/deps/libyrmcds/socket.c \
-    httpd/h2o/deps/libyrmcds/strerror.c \
-    httpd/h2o/deps/libyrmcds/text_mode.c \
-    httpd/h2o/deps/picohttpparser/picohttpparser.c \
-    httpd/h2o/lib/common/cache.c \
-    httpd/h2o/lib/common/file.c \
-    httpd/h2o/lib/common/filecache.c \
-    httpd/h2o/lib/common/hostinfo.c \
-    httpd/h2o/lib/common/http1client.c \
-    httpd/h2o/lib/common/memcached.c \
-    httpd/h2o/lib/common/memory.c \
-    httpd/h2o/lib/common/multithread.c \
-    httpd/h2o/lib/common/serverutil.c \
-    httpd/h2o/lib/common/socket.c \
-    httpd/h2o/lib/common/socketpool.c \
-    httpd/h2o/lib/common/string.c \
-    httpd/h2o/lib/common/time.c \
-    httpd/h2o/lib/common/timeout.c \
-    httpd/h2o/lib/common/url.c \
-    httpd/h2o/lib/core/config.c \
-    httpd/h2o/lib/core/configurator.c \
-    httpd/h2o/lib/core/context.c \
-    httpd/h2o/lib/core/headers.c \
-    httpd/h2o/lib/core/logconf.c \
-    httpd/h2o/lib/core/proxy.c \
-    httpd/h2o/lib/core/request.c \
-    httpd/h2o/lib/core/token.c \
-    httpd/h2o/lib/core/util.c \
-    httpd/h2o/lib/handler/access_log.c \
-    httpd/h2o/lib/handler/chunked.c \
-    httpd/h2o/lib/handler/compress.c \
-    httpd/h2o/lib/handler/compress/gzip.c \
-    httpd/h2o/lib/handler/errordoc.c \
-    httpd/h2o/lib/handler/expires.c \
-    httpd/h2o/lib/handler/fastcgi.c \
-    httpd/h2o/lib/handler/file.c \
-    httpd/h2o/lib/handler/headers.c \
-    httpd/h2o/lib/handler/mimemap.c \
-    httpd/h2o/lib/handler/proxy.c \
-    httpd/h2o/lib/handler/redirect.c \
-    httpd/h2o/lib/handler/reproxy.c \
-    httpd/h2o/lib/handler/throttle_resp.c \
-    httpd/h2o/lib/handler/status.c \
-    httpd/h2o/lib/handler/headers_util.c \
-    httpd/h2o/lib/handler/status/events.c \
-    httpd/h2o/lib/handler/status/requests.c \
-    httpd/h2o/lib/handler/http2_debug_state.c \
-    httpd/h2o/lib/handler/status/durations.c \
-    httpd/h2o/lib/handler/configurator/access_log.c \
-    httpd/h2o/lib/handler/configurator/compress.c \
-    httpd/h2o/lib/handler/configurator/errordoc.c \
-    httpd/h2o/lib/handler/configurator/expires.c \
-    httpd/h2o/lib/handler/configurator/fastcgi.c \
-    httpd/h2o/lib/handler/configurator/file.c \
-    httpd/h2o/lib/handler/configurator/headers.c \
-    httpd/h2o/lib/handler/configurator/proxy.c \
-    httpd/h2o/lib/handler/configurator/redirect.c \
-    httpd/h2o/lib/handler/configurator/reproxy.c \
-    httpd/h2o/lib/handler/configurator/throttle_resp.c \
-    httpd/h2o/lib/handler/configurator/status.c \
-    httpd/h2o/lib/handler/configurator/http2_debug_state.c \
-    httpd/h2o/lib/handler/configurator/headers_util.c \
-    httpd/h2o/lib/http1.c \
-    httpd/h2o/lib/tunnel.c \
-    httpd/h2o/lib/http2/cache_digests.c \
-    httpd/h2o/lib/http2/casper.c \
-    httpd/h2o/lib/http2/connection.c \
-    httpd/h2o/lib/http2/frame.c \
-    httpd/h2o/lib/http2/hpack.c \
-    httpd/h2o/lib/http2/scheduler.c \
-    httpd/h2o/lib/http2/stream.c \
-    httpd/h2o/lib/http2/http2_debug_state.c \
+    web/server/h2o/libh2o/deps/cloexec/cloexec.c \
+    web/server/h2o/libh2o/deps/libgkc/gkc.c \
+    web/server/h2o/libh2o/deps/libyrmcds/close.c \
+    web/server/h2o/libh2o/deps/libyrmcds/connect.c \
+    web/server/h2o/libh2o/deps/libyrmcds/recv.c \
+    web/server/h2o/libh2o/deps/libyrmcds/send.c \
+    web/server/h2o/libh2o/deps/libyrmcds/send_text.c \
+    web/server/h2o/libh2o/deps/libyrmcds/socket.c \
+    web/server/h2o/libh2o/deps/libyrmcds/strerror.c \
+    web/server/h2o/libh2o/deps/libyrmcds/text_mode.c \
+    web/server/h2o/libh2o/deps/picohttpparser/picohttpparser.c \
+    web/server/h2o/libh2o/lib/common/cache.c \
+    web/server/h2o/libh2o/lib/common/file.c \
+    web/server/h2o/libh2o/lib/common/filecache.c \
+    web/server/h2o/libh2o/lib/common/hostinfo.c \
+    web/server/h2o/libh2o/lib/common/http1client.c \
+    web/server/h2o/libh2o/lib/common/memcached.c \
+    web/server/h2o/libh2o/lib/common/memory.c \
+    web/server/h2o/libh2o/lib/common/multithread.c \
+    web/server/h2o/libh2o/lib/common/serverutil.c \
+    web/server/h2o/libh2o/lib/common/socket.c \
+    web/server/h2o/libh2o/lib/common/socketpool.c \
+    web/server/h2o/libh2o/lib/common/string.c \
+    web/server/h2o/libh2o/lib/common/time.c \
+    web/server/h2o/libh2o/lib/common/timeout.c \
+    web/server/h2o/libh2o/lib/common/url.c \
+    web/server/h2o/libh2o/lib/core/config.c \
+    web/server/h2o/libh2o/lib/core/configurator.c \
+    web/server/h2o/libh2o/lib/core/context.c \
+    web/server/h2o/libh2o/lib/core/headers.c \
+    web/server/h2o/libh2o/lib/core/logconf.c \
+    web/server/h2o/libh2o/lib/core/proxy.c \
+    web/server/h2o/libh2o/lib/core/request.c \
+    web/server/h2o/libh2o/lib/core/token.c \
+    web/server/h2o/libh2o/lib/core/util.c \
+    web/server/h2o/libh2o/lib/handler/access_log.c \
+    web/server/h2o/libh2o/lib/handler/chunked.c \
+    web/server/h2o/libh2o/lib/handler/compress.c \
+    web/server/h2o/libh2o/lib/handler/compress/gzip.c \
+    web/server/h2o/libh2o/lib/handler/errordoc.c \
+    web/server/h2o/libh2o/lib/handler/expires.c \
+    web/server/h2o/libh2o/lib/handler/fastcgi.c \
+    web/server/h2o/libh2o/lib/handler/file.c \
+    web/server/h2o/libh2o/lib/handler/headers.c \
+    web/server/h2o/libh2o/lib/handler/mimemap.c \
+    web/server/h2o/libh2o/lib/handler/proxy.c \
+    web/server/h2o/libh2o/lib/handler/redirect.c \
+    web/server/h2o/libh2o/lib/handler/reproxy.c \
+    web/server/h2o/libh2o/lib/handler/throttle_resp.c \
+    web/server/h2o/libh2o/lib/handler/status.c \
+    web/server/h2o/libh2o/lib/handler/headers_util.c \
+    web/server/h2o/libh2o/lib/handler/status/events.c \
+    web/server/h2o/libh2o/lib/handler/status/requests.c \
+    web/server/h2o/libh2o/lib/handler/http2_debug_state.c \
+    web/server/h2o/libh2o/lib/handler/status/durations.c \
+    web/server/h2o/libh2o/lib/handler/configurator/access_log.c \
+    web/server/h2o/libh2o/lib/handler/configurator/compress.c \
+    web/server/h2o/libh2o/lib/handler/configurator/errordoc.c \
+    web/server/h2o/libh2o/lib/handler/configurator/expires.c \
+    web/server/h2o/libh2o/lib/handler/configurator/fastcgi.c \
+    web/server/h2o/libh2o/lib/handler/configurator/file.c \
+    web/server/h2o/libh2o/lib/handler/configurator/headers.c \
+    web/server/h2o/libh2o/lib/handler/configurator/proxy.c \
+    web/server/h2o/libh2o/lib/handler/configurator/redirect.c \
+    web/server/h2o/libh2o/lib/handler/configurator/reproxy.c \
+    web/server/h2o/libh2o/lib/handler/configurator/throttle_resp.c \
+    web/server/h2o/libh2o/lib/handler/configurator/status.c \
+    web/server/h2o/libh2o/lib/handler/configurator/http2_debug_state.c \
+    web/server/h2o/libh2o/lib/handler/configurator/headers_util.c \
+    web/server/h2o/libh2o/lib/http1.c \
+    web/server/h2o/libh2o/lib/tunnel.c \
+    web/server/h2o/libh2o/lib/http2/cache_digests.c \
+    web/server/h2o/libh2o/lib/http2/casper.c \
+    web/server/h2o/libh2o/lib/http2/connection.c \
+    web/server/h2o/libh2o/lib/http2/frame.c \
+    web/server/h2o/libh2o/lib/http2/hpack.c \
+    web/server/h2o/libh2o/lib/http2/scheduler.c \
+    web/server/h2o/libh2o/lib/http2/stream.c \
+    web/server/h2o/libh2o/lib/http2/http2_debug_state.c \
     $(NULL)
 
 libh2o_a_INCLUDES = \
-    -I$(srcdir)/httpd/h2o/include \
-    -I$(srcdir)/httpd/h2o/deps/cloexec \
-    -I$(srcdir)/httpd/h2o/deps/brotli/enc \
-    -I$(srcdir)/httpd/h2o/deps/golombset \
-    -I$(srcdir)/httpd/h2o/deps/libgkc \
-    -I$(srcdir)/httpd/h2o/deps/libyrmcds \
-    -I$(srcdir)/httpd/h2o/deps/klib \
-    -I$(srcdir)/httpd/h2o/deps/neverbleed \
-    -I$(srcdir)/httpd/h2o/deps/picohttpparser \
-    -I$(srcdir)/httpd/h2o/deps/picotest \
-    -I$(srcdir)/httpd/h2o/deps/yaml/include \
-    -I$(srcdir)/httpd/h2o/deps/yoml \
+    -I$(srcdir)/web/server/h2o/libh2o/include \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/cloexec \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/brotli/enc \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/golombset \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/libgkc \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/libyrmcds \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/klib \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/neverbleed \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/picohttpparser \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/picotest \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/yaml/include \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/yoml \
     $(NULL)
 
-if ENABLE_HTTPD
+if ENABLE_H2O
 noinst_LIBRARIES += libh2o.a
 
 # until h2o updates support for OpenSSL 3.0 we silence the warnings
@@ -1058,7 +1062,7 @@ libh2o_a_CFLAGS = $(CFLAGS) -Wno-deprecated-declarations -Wno-unused-parameter -
 if LINUX
     libh2o_a_CFLAGS += -D_GNU_SOURCE
 endif
-endif #ENABLE_HTTPD
+endif #ENABLE_H2O
 
 NETDATA_FILES = \
     collectors/all.h \
@@ -1129,8 +1133,8 @@ if ENABLE_ACLK
     NETDATA_COMMON_LIBS += libmqttwebsockets.a
 endif
 
-if ENABLE_HTTPD
-    NETDATA_FILES += $(HTTPD_FILES)
+if ENABLE_H2O
+    NETDATA_FILES += $(H2O_FILES)
     NETDATA_COMMON_LIBS += libh2o.a
 endif
 
@@ -1205,6 +1209,13 @@ if ENABLE_PLUGIN_CGROUP_NETWORK
         $(NULL)
 endif
 
+if ENABLE_PLUGIN_LOCAL_LISTENERS
+    plugins_PROGRAMS += local-listeners
+    local_listeners_SOURCES = $(LOCAL_LISTENERS_FILES)
+    local_listeners_LDADD = \
+        $(NULL)
+endif
+
 if ENABLE_PLUGIN_FREEIPMI
     plugins_PROGRAMS += freeipmi.plugin
     freeipmi_plugin_SOURCES = $(FREEIPMI_PLUGIN_FILES)