diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-07-06 01:49:32 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-07-06 01:49:32 +0300 |
commit | c74bf56ee2910b5c90e5be2e31128580b85b9ca8 (patch) | |
tree | 6ef495ac5821ce2c4984d87feb65d79851dd38ec | |
parent | b45570d251980309e7c2d956dea8886f8aa13bdc (diff) |
Code reorg and cleanup - enrichment of /api/v2 (#15294)
* claim script now accepts the same params as the kickstart
* rewrote buildinfo to unify all methods
* added cloud unavailable in cloud status
* added all exporters
* renamed httpd to h2o
* rename ENABLE_COMPRESSION to ENABLE_LZ4
* rename global variable
* rename ENABLE_HTTPS to ENABLE_OPENSSL
* fix coverity-scan for openssl
* add lz4 to coverity-scan
* added all plugins and most of the features
* added all plugins and most of the features
* generalize bitmap code so that we can have any size of bitmaps
* cleanup
* fix compilation without protobuf
* fix compilation with others allocators
* fix bitmap
* comprehensive bitmaps unit test
* bitmap as macros
* added developer mode
* added system info to build info
* cloud available/unavailable
* added /api/v2/info
* added units and ni to transitions
* when showing instances and transitions, show only the instances that have transitions
* cleanup
* add missing quotes
* add anchor to transitions
* added more to build info
* calculate retention per tier and expose it to /api/v2/info
* added currently collected metrics
* do not show space and retention when no numbers are available
* fix impossible overflow
* Add function for transitions and execute callback
* In case of error, reset and try next dictionary entry
* Fix error message
* simpler logic to maintain retention per tier
* /api/v2/alert_transitions
* Handle case of recipient null
Convert after and before to usec
* Add classification, type and component
* working /api/v2/alert_transitions
* Fix query to properly handle context and alert name
* cleanup
* Add search with transition
* accept transition in /api/v2/alert_transitions
* totaly dynamic facets
* fixed debug info
* restructured facets
* cleanup; removal of options=transitions
* updated alert entries flags
* method to exec
* Return also exec run timestamp
Temp table cleanup only when we don't execute with a transition
* cleanup obsolete anchor parameter
* Add sql_get_alert_configuration function
* added options=config to alert_transitions
* added /api/v2/alert_config
* preliminary work for /api/v2/claim
* initialize variables; do not expose expected retention if no disk space info is available; do not report aclk as initializing when not claimed
* fix claim session key filename
* put a newline into the session key file
* more progress on claiming
* final /api/v2/claim endpoint
* after claiming, refresh our state at the output
* Fix query to fetch config
* Remove debug log
* add configuration objects
* add configuration objects - fixed
* respect the NETDATA_DISABLE_CLOUD env variable
* NETDATA_DISABLE_CLOUD env variable sets the default, but the config sets the final value
* use a new claimed_id on every claiming
* regenerate random key on claiming and wait for online status
* ignore write() return value when writing a newline
* dont show cloud status disabled when claimed_id is missing
* added ctx to alert instances
* cleanup config and transitions from /api/v2/alerts
* fix unused variable
* in /api/v2/alert_config show 1 config without an array
* show alert values conditionally, by appending options=values
* When storing host info if the key value is empty, store unknown
* added options=summary to control when the alerts summary is shown
* increased http_api_v2 to version 5
* claming random key file is now not world readable
* added local-listeners binary that detects all the listening ports, their IPs and their command lines
---------
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
62 files changed, 3965 insertions, 1327 deletions
diff --git a/.gitignore b/.gitignore index ec56d649dc..dad6afcb44 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,9 @@ slabinfo.plugin cgroup-network !cgroup-network/ +local-listeners +!local-listeners/ + ebpf.plugin collectors/ebpf.plugin/reset_netdata_trace.sh !ebpf.plugin/ diff --git a/.gitmodules b/.gitmodules index d2730eb8a4..2dae4a1dd2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ url = https://github.com/davisking/dlib.git shallow = true ignore = dirty -[submodule "httpd/h2o"] - path = httpd/h2o +[submodule "web/server/h2o/libh2o"] + path = web/server/h2o/libh2o url = https://github.com/h2o/h2o.git ignore = untracked diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bda2638b6..866cb5d8ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,7 +102,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4) set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER}) set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES}) set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS}) -# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1") +# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_LZ4=1") # ----------------------------------------------------------------------------- # Judy General purpose dynamic array diff --git a/Makefile.am b/Makefile.am index d73ad972b0..c6f55d4dc9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,7 +84,7 @@ dist_noinst_DATA = \ packaging/protobuf.version \ packaging/version \ database/engine/journalfile_v2.ksy.in \ - httpd/h2o \ + web/server/h2o/libh2o \ $(NULL) # until integrated within build @@ -281,6 +281,10 @@ CGROUP_NETWORK_FILES = \ $(LIBNETDATA_FILES) \ $(NULL) +LOCAL_LISTENERS_FILES = \ + collectors/plugins.d/local_listeners.c \ + $(NULL) + DISKSPACE_PLUGIN_FILES = \ collectors/diskspace.plugin/plugin_diskspace.c \ $(NULL) @@ -945,111 +949,111 @@ DAEMON_FILES = \ daemon/unit_test.h \ $(NULL) -HTTPD_FILES = \ - httpd/http_server.c \ - httpd/http_server.h \ - httpd/h2o_utils.c \ - httpd/h2o_utils.h \ +H2O_FILES = \ + web/server/h2o/http_server.c \ + web/server/h2o/http_server.h \ + web/server/h2o/h2o_utils.c \ + web/server/h2o/h2o_utils.h \ $(NULL) libh2o_a_SOURCES = \ - httpd/h2o/deps/cloexec/cloexec.c \ - httpd/h2o/deps/libgkc/gkc.c \ - httpd/h2o/deps/libyrmcds/close.c \ - httpd/h2o/deps/libyrmcds/connect.c \ - httpd/h2o/deps/libyrmcds/recv.c \ - httpd/h2o/deps/libyrmcds/send.c \ - httpd/h2o/deps/libyrmcds/send_text.c \ - httpd/h2o/deps/libyrmcds/socket.c \ - httpd/h2o/deps/libyrmcds/strerror.c \ - httpd/h2o/deps/libyrmcds/text_mode.c \ - httpd/h2o/deps/picohttpparser/picohttpparser.c \ - httpd/h2o/lib/common/cache.c \ - httpd/h2o/lib/common/file.c \ - httpd/h2o/lib/common/filecache.c \ - httpd/h2o/lib/common/hostinfo.c \ - httpd/h2o/lib/common/http1client.c \ - httpd/h2o/lib/common/memcached.c \ - httpd/h2o/lib/common/memory.c \ - httpd/h2o/lib/common/multithread.c \ - httpd/h2o/lib/common/serverutil.c \ - httpd/h2o/lib/common/socket.c \ - httpd/h2o/lib/common/socketpool.c \ - httpd/h2o/lib/common/string.c \ - httpd/h2o/lib/common/time.c \ - httpd/h2o/lib/common/timeout.c \ - httpd/h2o/lib/common/url.c \ - httpd/h2o/lib/core/config.c \ - httpd/h2o/lib/core/configurator.c \ - httpd/h2o/lib/core/context.c \ - httpd/h2o/lib/core/headers.c \ - httpd/h2o/lib/core/logconf.c \ - httpd/h2o/lib/core/proxy.c \ - httpd/h2o/lib/core/request.c \ - httpd/h2o/lib/core/token.c \ - httpd/h2o/lib/core/util.c \ - httpd/h2o/lib/handler/access_log.c \ - httpd/h2o/lib/handler/chunked.c \ - httpd/h2o/lib/handler/compress.c \ - httpd/h2o/lib/handler/compress/gzip.c \ - httpd/h2o/lib/handler/errordoc.c \ - httpd/h2o/lib/handler/expires.c \ - httpd/h2o/lib/handler/fastcgi.c \ - httpd/h2o/lib/handler/file.c \ - httpd/h2o/lib/handler/headers.c \ - httpd/h2o/lib/handler/mimemap.c \ - httpd/h2o/lib/handler/proxy.c \ - httpd/h2o/lib/handler/redirect.c \ - httpd/h2o/lib/handler/reproxy.c \ - httpd/h2o/lib/handler/throttle_resp.c \ - httpd/h2o/lib/handler/status.c \ - httpd/h2o/lib/handler/headers_util.c \ - httpd/h2o/lib/handler/status/events.c \ - httpd/h2o/lib/handler/status/requests.c \ - httpd/h2o/lib/handler/http2_debug_state.c \ - httpd/h2o/lib/handler/status/durations.c \ - httpd/h2o/lib/handler/configurator/access_log.c \ - httpd/h2o/lib/handler/configurator/compress.c \ - httpd/h2o/lib/handler/configurator/errordoc.c \ - httpd/h2o/lib/handler/configurator/expires.c \ - httpd/h2o/lib/handler/configurator/fastcgi.c \ - httpd/h2o/lib/handler/configurator/file.c \ - httpd/h2o/lib/handler/configurator/headers.c \ - httpd/h2o/lib/handler/configurator/proxy.c \ - httpd/h2o/lib/handler/configurator/redirect.c \ - httpd/h2o/lib/handler/configurator/reproxy.c \ - httpd/h2o/lib/handler/configurator/throttle_resp.c \ - httpd/h2o/lib/handler/configurator/status.c \ - httpd/h2o/lib/handler/configurator/http2_debug_state.c \ - httpd/h2o/lib/handler/configurator/headers_util.c \ - httpd/h2o/lib/http1.c \ - httpd/h2o/lib/tunnel.c \ - httpd/h2o/lib/http2/cache_digests.c \ - httpd/h2o/lib/http2/casper.c \ - httpd/h2o/lib/http2/connection.c \ - httpd/h2o/lib/http2/frame.c \ - httpd/h2o/lib/http2/hpack.c \ - httpd/h2o/lib/http2/scheduler.c \ - httpd/h2o/lib/http2/stream.c \ - httpd/h2o/lib/http2/http2_debug_state.c \ + web/server/h2o/libh2o/deps/cloexec/cloexec.c \ + web/server/h2o/libh2o/deps/libgkc/gkc.c \ + web/server/h2o/libh2o/deps/libyrmcds/close.c \ + web/server/h2o/libh2o/deps/libyrmcds/connect.c \ + web/server/h2o/libh2o/deps/libyrmcds/recv.c \ + web/server/h2o/libh2o/deps/libyrmcds/send.c \ + web/server/h2o/libh2o/deps/libyrmcds/send_text.c \ + web/server/h2o/libh2o/deps/libyrmcds/socket.c \ + web/server/h2o/libh2o/deps/libyrmcds/strerror.c \ + web/server/h2o/libh2o/deps/libyrmcds/text_mode.c \ + web/server/h2o/libh2o/deps/picohttpparser/picohttpparser.c \ + web/server/h2o/libh2o/lib/common/cache.c \ + web/server/h2o/libh2o/lib/common/file.c \ + web/server/h2o/libh2o/lib/common/filecache.c \ + web/server/h2o/libh2o/lib/common/hostinfo.c \ + web/server/h2o/libh2o/lib/common/http1client.c \ + web/server/h2o/libh2o/lib/common/memcached.c \ + web/server/h2o/libh2o/lib/common/memory.c \ + web/server/h2o/libh2o/lib/common/multithread.c \ + web/server/h2o/libh2o/lib/common/serverutil.c \ + web/server/h2o/libh2o/lib/common/socket.c \ + web/server/h2o/libh2o/lib/common/socketpool.c \ + web/server/h2o/libh2o/lib/common/string.c \ + web/server/h2o/libh2o/lib/common/time.c \ + web/server/h2o/libh2o/lib/common/timeout.c \ + web/server/h2o/libh2o/lib/common/url.c \ + web/server/h2o/libh2o/lib/core/config.c \ + web/server/h2o/libh2o/lib/core/configurator.c \ + web/server/h2o/libh2o/lib/core/context.c \ + web/server/h2o/libh2o/lib/core/headers.c \ + web/server/h2o/libh2o/lib/core/logconf.c \ + web/server/h2o/libh2o/lib/core/proxy.c \ + web/server/h2o/libh2o/lib/core/request.c \ + web/server/h2o/libh2o/lib/core/token.c \ + web/server/h2o/libh2o/lib/core/util.c \ + web/server/h2o/libh2o/lib/handler/access_log.c \ + web/server/h2o/libh2o/lib/handler/chunked.c \ + web/server/h2o/libh2o/lib/handler/compress.c \ + web/server/h2o/libh2o/lib/handler/compress/gzip.c \ + web/server/h2o/libh2o/lib/handler/errordoc.c \ + web/server/h2o/libh2o/lib/handler/expires.c \ + web/server/h2o/libh2o/lib/handler/fastcgi.c \ + web/server/h2o/libh2o/lib/handler/file.c \ + web/server/h2o/libh2o/lib/handler/headers.c \ + web/server/h2o/libh2o/lib/handler/mimemap.c \ + web/server/h2o/libh2o/lib/handler/proxy.c \ + web/server/h2o/libh2o/lib/handler/redirect.c \ + web/server/h2o/libh2o/lib/handler/reproxy.c \ + web/server/h2o/libh2o/lib/handler/throttle_resp.c \ + web/server/h2o/libh2o/lib/handler/status.c \ + web/server/h2o/libh2o/lib/handler/headers_util.c \ + web/server/h2o/libh2o/lib/handler/status/events.c \ + web/server/h2o/libh2o/lib/handler/status/requests.c \ + web/server/h2o/libh2o/lib/handler/http2_debug_state.c \ + web/server/h2o/libh2o/lib/handler/status/durations.c \ + web/server/h2o/libh2o/lib/handler/configurator/access_log.c \ + web/server/h2o/libh2o/lib/handler/configurator/compress.c \ + web/server/h2o/libh2o/lib/handler/configurator/errordoc.c \ + web/server/h2o/libh2o/lib/handler/configurator/expires.c \ + web/server/h2o/libh2o/lib/handler/configurator/fastcgi.c \ + web/server/h2o/libh2o/lib/handler/configurator/file.c \ + web/server/h2o/libh2o/lib/handler/configurator/headers.c \ + web/server/h2o/libh2o/lib/handler/configurator/proxy.c \ + web/server/h2o/libh2o/lib/handler/configurator/redirect.c \ + web/server/h2o/libh2o/lib/handler/configurator/reproxy.c \ + web/server/h2o/libh2o/lib/handler/configurator/throttle_resp.c \ + web/server/h2o/libh2o/lib/handler/configurator/status.c \ + web/server/h2o/libh2o/lib/handler/configurator/http2_debug_state.c \ + web/server/h2o/libh2o/lib/handler/configurator/headers_util.c \ + web/server/h2o/libh2o/lib/http1.c \ + web/server/h2o/libh2o/lib/tunnel.c \ + web/server/h2o/libh2o/lib/http2/cache_digests.c \ + web/server/h2o/libh2o/lib/http2/casper.c \ + web/server/h2o/libh2o/lib/http2/connection.c \ + web/server/h2o/libh2o/lib/http2/frame.c \ + web/server/h2o/libh2o/lib/http2/hpack.c \ + web/server/h2o/libh2o/lib/http2/scheduler.c \ + web/server/h2o/libh2o/lib/http2/stream.c \ + web/server/h2o/libh2o/lib/http2/http2_debug_state.c \ $(NULL) libh2o_a_INCLUDES = \ - -I$(srcdir)/httpd/h2o/include \ - -I$(srcdir)/httpd/h2o/deps/cloexec \ - -I$(srcdir)/httpd/h2o/deps/brotli/enc \ - -I$(srcdir)/httpd/h2o/deps/golombset \ - -I$(srcdir)/httpd/h2o/deps/libgkc \ - -I$(srcdir)/httpd/h2o/deps/libyrmcds \ - -I$(srcdir)/httpd/h2o/deps/klib \ - -I$(srcdir)/httpd/h2o/deps/neverbleed \ - -I$(srcdir)/httpd/h2o/deps/picohttpparser \ - -I$(srcdir)/httpd/h2o/deps/picotest \ - -I$(srcdir)/httpd/h2o/deps/yaml/include \ - -I$(srcdir)/httpd/h2o/deps/yoml \ + -I$(srcdir)/web/server/h2o/libh2o/include \ + -I$(srcdir)/web/server/h2o/libh2o/deps/cloexec \ + -I$(srcdir)/web/server/h2o/libh2o/deps/brotli/enc \ + -I$(srcdir)/web/server/h2o/libh2o/deps/golombset \ + -I$(srcdir)/web/server/h2o/libh2o/deps/libgkc \ + -I$(srcdir)/web/server/h2o/libh2o/deps/libyrmcds \ + -I$(srcdir)/web/server/h2o/libh2o/deps/klib \ + -I$(srcdir)/web/server/h2o/libh2o/deps/neverbleed \ + -I$(srcdir)/web/server/h2o/libh2o/deps/picohttpparser \ + -I$(srcdir)/web/server/h2o/libh2o/deps/picotest \ + -I$(srcdir)/web/server/h2o/libh2o/deps/yaml/include \ + -I$(srcdir)/web/server/h2o/libh2o/deps/yoml \ $(NULL) -if ENABLE_HTTPD +if ENABLE_H2O noinst_LIBRARIES += libh2o.a # until h2o updates support for OpenSSL 3.0 we silence the warnings @@ -1058,7 +1062,7 @@ libh2o_a_CFLAGS = $(CFLAGS) -Wno-deprecated-declarations -Wno-unused-parameter - if LINUX libh2o_a_CFLAGS += -D_GNU_SOURCE endif -endif #ENABLE_HTTPD +endif #ENABLE_H2O NETDATA_FILES = \ collectors/all.h \ @@ -1129,8 +1133,8 @@ if ENABLE_ACLK NETDATA_COMMON_LIBS += libmqttwebsockets.a endif -if ENABLE_HTTPD - NETDATA_FILES += $(HTTPD_FILES) +if ENABLE_H2O + NETDATA_FILES += $(H2O_FILES) NETDATA_COMMON_LIBS += libh2o.a endif @@ -1205,6 +1209,13 @@ if ENABLE_PLUGIN_CGROUP_NETWORK $(NULL) endif +if ENABLE_PLUGIN_LOCAL_LISTENERS + plugins_PROGRAMS += local-listeners + local_listeners_SOURCES = $(LOCAL_LISTENERS_FILES) + local_listeners_LDADD = \ + $(NULL) +endif + if ENABLE_PLUGIN_FREEIPMI plugins_PROGRAMS += freeipmi.plugin freeipmi_plugin_SOURCES = $(FREEIPMI_PLUGIN_FILES) diff --git a/aclk/aclk.c b/aclk/aclk.c index 3ff5515684..b7b6a3d81c 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -489,15 +489,15 @@ static int aclk_get_transport_idx(aclk_env_t *env) { } #endif -ACLK_STATUS aclk_status = ACLK_STATUS_INITIALIZING; +ACLK_STATUS aclk_status = ACLK_STATUS_NONE; const char *aclk_status_to_string(void) { switch(aclk_status) { case ACLK_STATUS_CONNECTED: return "connected"; - case ACLK_STATUS_INITIALIZING: - return "initializing"; + case ACLK_STATUS_NONE: + return "none"; case ACLK_STATUS_DISABLED: return "disabled"; diff --git a/aclk/aclk.h b/aclk/aclk.h index 17353be29d..0badc1a628 100644 --- a/aclk/aclk.h +++ b/aclk/aclk.h @@ -15,7 +15,7 @@ typedef enum __attribute__((packed)) { ACLK_STATUS_CONNECTED = 0, - ACLK_STATUS_INITIALIZING, + ACLK_STATUS_NONE, ACLK_STATUS_DISABLED, ACLK_STATUS_NO_CLOUD_URL, ACLK_STATUS_INVALID_CLOUD_URL, diff --git a/aclk/aclk_capas.c b/aclk/aclk_capas.c index a897d30446..a81116faf6 100644 --- a/aclk/aclk_capas.c +++ b/aclk/aclk_capas.c @@ -4,7 +4,7 @@ #include "ml/ml.h" -#define HTTP_API_V2_VERSION 4 +#define HTTP_API_V2_VERSION 5 const struct capability *aclk_get_agent_capas() { diff --git a/claim/claim.c b/claim/claim.c index 48ffcc4c72..1f61fdee28 100644 --- a/claim/claim.c +++ b/claim/claim.c @@ -42,16 +42,16 @@ char *get_agent_claimid() } #define CLAIMING_COMMAND_LENGTH 16384 -#define CLAIMING_PROXY_LENGTH CLAIMING_COMMAND_LENGTH/4 +#define CLAIMING_PROXY_LENGTH (CLAIMING_COMMAND_LENGTH/4) extern struct registry registry; /* rrd_init() and post_conf_load() must have been called before this function */ -void claim_agent(char *claiming_arguments) +CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg) { - if (!netdata_cloud_enabled) { + if (!force || !netdata_cloud_enabled) { error("Refusing to claim agent -> cloud functionality has been disabled"); - return; + return CLAIM_AGENT_CLOUD_DISABLED; } #ifndef DISABLE_CLOUD @@ -62,8 +62,11 @@ void claim_agent(char *claiming_arguments) // This is guaranteed to be set early in main via post_conf_load() char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL); - if (cloud_base_url == NULL) - fatal("Do not move the cloud base url out of post_conf_load!!"); + if (cloud_base_url == NULL) { + internal_fatal(true, "Do not move the cloud base url out of post_conf_load!!"); + return CLAIM_AGENT_NO_CLOUD_URL; + } + const char *proxy_str; ACLK_PROXY_TYPE proxy_type; char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy"; @@ -76,7 +79,6 @@ void claim_agent(char *claiming_arguments) snprintfz(command_buffer, CLAIMING_COMMAND_LENGTH, "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s", - proxy_flag, netdata_configured_hostname, localhost->machine_guid, @@ -87,7 +89,7 @@ void claim_agent(char *claiming_arguments) fp_child_output = netdata_popen(command_buffer, &command_pid, &fp_child_input); if(!fp_child_output) { error("Cannot popen(\"%s\").", command_buffer); - return; + return CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT; } netdata_log_info("Waiting for claiming command to finish."); while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp_child_output) != NULL) {;} @@ -95,25 +97,31 @@ void claim_agent(char *claiming_arguments) netdata_log_info("Agent claiming command returned with code %d", exit_code); if (0 == exit_code) { load_claiming_state(); - return; + return CLAIM_AGENT_OK; } if (exit_code < 0) { error("Agent claiming command failed to complete its run."); - return; + return CLAIM_AGENT_CLAIM_SCRIPT_FAILED; } errno = 0; unsigned maximum_known_exit_code = sizeof(claiming_errors) / sizeof(claiming_errors[0]) - 1; if ((unsigned)exit_code > maximum_known_exit_code) { error("Agent fai |