diff options
62 files changed, 3965 insertions, 1327 deletions
diff --git a/.gitignore b/.gitignore index ec56d649dc..dad6afcb44 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,9 @@ slabinfo.plugin cgroup-network !cgroup-network/ +local-listeners +!local-listeners/ + ebpf.plugin collectors/ebpf.plugin/reset_netdata_trace.sh !ebpf.plugin/ diff --git a/.gitmodules b/.gitmodules index d2730eb8a4..2dae4a1dd2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ url = https://github.com/davisking/dlib.git shallow = true ignore = dirty -[submodule "httpd/h2o"] - path = httpd/h2o +[submodule "web/server/h2o/libh2o"] + path = web/server/h2o/libh2o url = https://github.com/h2o/h2o.git ignore = untracked diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bda2638b6..866cb5d8ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,7 +102,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4) set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER}) set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES}) set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS}) -# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1") +# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_LZ4=1") # ----------------------------------------------------------------------------- # Judy General purpose dynamic array diff --git a/Makefile.am b/Makefile.am index d73ad972b0..c6f55d4dc9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,7 +84,7 @@ dist_noinst_DATA = \ packaging/protobuf.version \ packaging/version \ database/engine/journalfile_v2.ksy.in \ - httpd/h2o \ + web/server/h2o/libh2o \ $(NULL) # until integrated within build @@ -281,6 +281,10 @@ CGROUP_NETWORK_FILES = \ $(LIBNETDATA_FILES) \ $(NULL) +LOCAL_LISTENERS_FILES = \ + collectors/plugins.d/local_listeners.c \ + $(NULL) + DISKSPACE_PLUGIN_FILES = \ collectors/diskspace.plugin/plugin_diskspace.c \ $(NULL) @@ -945,111 +949,111 @@ DAEMON_FILES = \ daemon/unit_test.h \ $(NULL) -HTTPD_FILES = \ - httpd/http_server.c \ - httpd/http_server.h \ - httpd/h2o_utils.c \ - httpd/h2o_utils.h \ +H2O_FILES = \ + web/server/h2o/http_server.c \ + web/server/h2o/http_server.h \ + web/server/h2o/h2o_utils.c \ + web/server/h2o/h2o_utils.h \ $(NULL) libh2o_a_SOURCES = \ - httpd/h2o/deps/cloexec/cloexec.c \ - httpd/h2o/deps/libgkc/gkc.c \ - httpd/h2o/deps/libyrmcds/close.c \ - httpd/h2o/deps/libyrmcds/connect.c \ - httpd/h2o/deps/libyrmcds/recv.c \ - httpd/h2o/deps/libyrmcds/send.c \ - httpd/h2o/deps/libyrmcds/send_text.c \ - httpd/h2o/deps/libyrmcds/socket.c \ - httpd/h2o/deps/libyrmcds/strerror.c \ - httpd/h2o/deps/libyrmcds/text_mode.c \ - httpd/h2o/deps/picohttpparser/picohttpparser.c \ - httpd/h2o/lib/common/cache.c \ - httpd/h2o/lib/common/file.c \ - httpd/h2o/lib/common/filecache.c \ - httpd/h2o/lib/common/hostinfo.c \ - httpd/h2o/lib/common/http1client.c \ - httpd/h2o/lib/common/memcached.c \ - httpd/h2o/lib/common/memory.c \ - httpd/h2o/lib/common/multithread.c \ - httpd/h2o/lib/common/serverutil.c \ - httpd/h2o/lib/common/socket.c \ - httpd/h2o/lib/common/socketpool.c \ - httpd/h2o/lib/common/string.c \ - httpd/h2o/lib/common/time.c \ - httpd/h2o/lib/common/timeout.c \ - httpd/h2o/lib/common/url.c \ - httpd/h2o/lib/core/config.c \ - httpd/h2o/lib/core/configurator.c \ - httpd/h2o/lib/core/context.c \ - httpd/h2o/lib/core/headers.c \ - httpd/h2o/lib/core/logconf.c \ - httpd/h2o/lib/core/proxy.c \ - httpd/h2o/lib/core/request.c \ - httpd/h2o/lib/core/token.c \ - httpd/h2o/lib/core/util.c \ - httpd/h2o/lib/handler/access_log.c \ - httpd/h2o/lib/handler/chunked.c \ - httpd/h2o/lib/handler/compress.c \ - httpd/h2o/lib/handler/compress/gzip.c \ - httpd/h2o/lib/handler/errordoc.c \ - httpd/h2o/lib/handler/expires.c \ - httpd/h2o/lib/handler/fastcgi.c \ - httpd/h2o/lib/handler/file.c \ - httpd/h2o/lib/handler/headers.c \ - httpd/h2o/lib/handler/mimemap.c \ - httpd/h2o/lib/handler/proxy.c \ - httpd/h2o/lib/handler/redirect.c \ - httpd/h2o/lib/handler/reproxy.c \ - httpd/h2o/lib/handler/throttle_resp.c \ - httpd/h2o/lib/handler/status.c \ - httpd/h2o/lib/handler/headers_util.c \ - httpd/h2o/lib/handler/status/events.c \ - httpd/h2o/lib/handler/status/requests.c \ - httpd/h2o/lib/handler/http2_debug_state.c \ - httpd/h2o/lib/handler/status/durations.c \ - httpd/h2o/lib/handler/configurator/access_log.c \ - httpd/h2o/lib/handler/configurator/compress.c \ - httpd/h2o/lib/handler/configurator/errordoc.c \ - httpd/h2o/lib/handler/configurator/expires.c \ - httpd/h2o/lib/handler/configurator/fastcgi.c \ - httpd/h2o/lib/handler/configurator/file.c \ - httpd/h2o/lib/handler/configurator/headers.c \ - httpd/h2o/lib/handler/configurator/proxy.c \ - httpd/h2o/lib/handler/configurator/redirect.c \ - httpd/h2o/lib/handler/configurator/reproxy.c \ - httpd/h2o/lib/handler/configurator/throttle_resp.c \ - httpd/h2o/lib/handler/configurator/status.c \ - httpd/h2o/lib/handler/configurator/http2_debug_state.c \ - httpd/h2o/lib/handler/configurator/headers_util.c \ - httpd/h2o/lib/http1.c \ - httpd/h2o/lib/tunnel.c \ - httpd/h2o/lib/http2/cache_digests.c \ - httpd/h2o/lib/http2/casper.c \ - httpd/h2o/lib/http2/connection.c \ - httpd/h2o/lib/http2/frame.c \ - httpd/h2o/lib/http2/hpack.c \ - httpd/h2o/lib/http2/scheduler.c \ - httpd/h2o/lib/http2/stream.c \ - httpd/h2o/lib/http2/http2_debug_state.c \ + web/server/h2o/libh2o/deps/cloexec/cloexec.c \ + web/server/h2o/libh2o/deps/libgkc/gkc.c \ + web/server/h2o/libh2o/deps/libyrmcds/close.c \ + web/server/h2o/libh2o/deps/libyrmcds/connect.c \ + web/server/h2o/libh2o/deps/libyrmcds/recv.c \ + web/server/h2o/libh2o/deps/libyrmcds/send.c \ + web/server/h2o/libh2o/deps/libyrmcds/send_text.c \ + web/server/h2o/libh2o/deps/libyrmcds/socket.c \ + web/server/h2o/libh2o/deps/libyrmcds/strerror.c \ + web/server/h2o/libh2o/deps/libyrmcds/text_mode.c \ + web/server/h2o/libh2o/deps/picohttpparser/picohttpparser.c \ + web/server/h2o/libh2o/lib/common/cache.c \ + web/server/h2o/libh2o/lib/common/file.c \ + web/server/h2o/libh2o/lib/common/filecache.c \ + web/server/h2o/libh2o/lib/common/hostinfo.c \ + web/server/h2o/libh2o/lib/common/http1client.c \ + web/server/h2o/libh2o/lib/common/memcached.c \ + web/server/h2o/libh2o/lib/common/memory.c \ + web/server/h2o/libh2o/lib/common/multithread.c \ + web/server/h2o/libh2o/lib/common/serverutil.c \ + web/server/h2o/libh2o/lib/common/socket.c \ + web/server/h2o/libh2o/lib/common/socketpool.c \ + web/server/h2o/libh2o/lib/common/string.c \ + web/server/h2o/libh2o/lib/common/time.c \ + web/server/h2o/libh2o/lib/common/timeout.c \ + web/server/h2o/libh2o/lib/common/url.c \ + web/server/h2o/libh2o/lib/core/config.c \ + web/server/h2o/libh2o/lib/core/configurator.c \ + web/server/h2o/libh2o/lib/core/context.c \ + web/server/h2o/libh2o/lib/core/headers.c \ + web/server/h2o/libh2o/lib/core/logconf.c \ + web/server/h2o/libh2o/lib/core/proxy.c \ + web/server/h2o/libh2o/lib/core/request.c \ + web/server/h2o/libh2o/lib/core/token.c \ + web/server/h2o/libh2o/lib/core/util.c \ + web/server/h2o/libh2o/lib/handler/access_log.c \ + web/server/h2o/libh2o/lib/handler/chunked.c \ + web/server/h2o/libh2o/lib/handler/compress.c \ + web/server/h2o/libh2o/lib/handler/compress/gzip.c \ + web/server/h2o/libh2o/lib/handler/errordoc.c \ + web/server/h2o/libh2o/lib/handler/expires.c \ + web/server/h2o/libh2o/lib/handler/fastcgi.c \ + web/server/h2o/libh2o/lib/handler/file.c \ + web/server/h2o/libh2o/lib/handler/headers.c \ + web/server/h2o/libh2o/lib/handler/mimemap.c \ + web/server/h2o/libh2o/lib/handler/proxy.c \ + web/server/h2o/libh2o/lib/handler/redirect.c \ + web/server/h2o/libh2o/lib/handler/reproxy.c \ + web/server/h2o/libh2o/lib/handler/throttle_resp.c \ + web/server/h2o/libh2o/lib/handler/status.c \ + web/server/h2o/libh2o/lib/handler/headers_util.c \ + web/server/h2o/libh2o/lib/handler/status/events.c \ + web/server/h2o/libh2o/lib/handler/status/requests.c \ + web/server/h2o/libh2o/lib/handler/http2_debug_state.c \ + web/server/h2o/libh2o/lib/handler/status/durations.c \ + web/server/h2o/libh2o/lib/handler/configurator/access_log.c \ + web/server/h2o/libh2o/lib/handler/configurator/compress.c \ + web/server/h2o/libh2o/lib/handler/configurator/errordoc.c \ + web/server/h2o/libh2o/lib/handler/configurator/expires.c \ + web/server/h2o/libh2o/lib/handler/configurator/fastcgi.c \ + web/server/h2o/libh2o/lib/handler/configurator/file.c \ + web/server/h2o/libh2o/lib/handler/configurator/headers.c \ + web/server/h2o/libh2o/lib/handler/configurator/proxy.c \ + web/server/h2o/libh2o/lib/handler/configurator/redirect.c \ + web/server/h2o/libh2o/lib/handler/configurator/reproxy.c \ + web/server/h2o/libh2o/lib/handler/configurator/throttle_resp.c \ + web/server/h2o/libh2o/lib/handler/configurator/status.c \ + web/server/h2o/libh2o/lib/handler/configurator/http2_debug_state.c \ + web/server/h2o/libh2o/lib/handler/configurator/headers_util.c \ + web/server/h2o/libh2o/lib/http1.c \ + web/server/h2o/libh2o/lib/tunnel.c \ + web/server/h2o/libh2o/lib/http2/cache_digests.c \ + web/server/h2o/libh2o/lib/http2/casper.c \ + web/server/h2o/libh2o/lib/http2/connection.c \ + web/server/h2o/libh2o/lib/http2/frame.c \ + web/server/h2o/libh2o/lib/http2/hpack.c \ + web/server/h2o/libh2o/lib/http2/scheduler.c \ + web/server/h2o/libh2o/lib/http2/stream.c \ + web/server/h2o/libh2o/lib/http2/http2_debug_state.c \ $(NULL) libh2o_a_INCLUDES = \ - -I$(srcdir)/httpd/h2o/include \ - -I$(srcdir)/httpd/h2o/deps/cloexec \ - -I$(srcdir)/httpd/h2o/deps/brotli/enc \ - -I$(srcdir)/httpd/h2o/deps/golombset \ - -I$(srcdir)/httpd/h2o/deps/libgkc \ - -I$(srcdir)/httpd/h2o/deps/libyrmcds \ - -I$(srcdir)/httpd/h2o/deps/klib \ - -I$(srcdir)/httpd/h2o/deps/neverbleed \ - -I$(srcdir)/httpd/h2o/deps/picohttpparser \ - -I$(srcdir)/httpd/h2o/deps/picotest \ - -I$(srcdir)/httpd/h2o/deps/yaml/include \ - -I$(srcdir)/httpd/h2o/deps/yoml \ + -I$(srcdir)/web/server/h2o/libh2o/include \ + -I$(srcdir)/web/server/h2o/libh2o/deps/cloexec \ + -I$(srcdir)/web/server/h2o/libh2o/deps/brotli/enc \ + -I$(srcdir)/web/server/h2o/libh2o/deps/golombset \ + -I$(srcdir)/web/server/h2o/libh2o/deps/libgkc \ + -I$(srcdir)/web/server/h2o/libh2o/deps/libyrmcds \ + -I$(srcdir)/web/server/h2o/libh2o/deps/klib \ + -I$(srcdir)/web/server/h2o/libh2o/deps/neverbleed \ + -I$(srcdir)/web/server/h2o/libh2o/deps/picohttpparser \ + -I$(srcdir)/web/server/h2o/libh2o/deps/picotest \ + -I$(srcdir)/web/server/h2o/libh2o/deps/yaml/include \ + -I$(srcdir)/web/server/h2o/libh2o/deps/yoml \ $(NULL) -if ENABLE_HTTPD +if ENABLE_H2O noinst_LIBRARIES += libh2o.a # until h2o updates support for OpenSSL 3.0 we silence the warnings @@ -1058,7 +1062,7 @@ libh2o_a_CFLAGS = $(CFLAGS) -Wno-deprecated-declarations -Wno-unused-parameter - if LINUX libh2o_a_CFLAGS += -D_GNU_SOURCE endif -endif #ENABLE_HTTPD +endif #ENABLE_H2O NETDATA_FILES = \ collectors/all.h \ @@ -1129,8 +1133,8 @@ if ENABLE_ACLK NETDATA_COMMON_LIBS += libmqttwebsockets.a endif -if ENABLE_HTTPD - NETDATA_FILES += $(HTTPD_FILES) +if ENABLE_H2O + NETDATA_FILES += $(H2O_FILES) NETDATA_COMMON_LIBS += libh2o.a endif @@ -1205,6 +1209,13 @@ if ENABLE_PLUGIN_CGROUP_NETWORK $(NULL) endif +if ENABLE_PLUGIN_LOCAL_LISTENERS + plugins_PROGRAMS += local-listeners + local_listeners_SOURCES = $(LOCAL_LISTENERS_FILES) + local_listeners_LDADD = \ + $(NULL) +endif + if ENABLE_PLUGIN_FREEIPMI plugins_PROGRAMS += freeipmi.plugin freeipmi_plugin_SOURCES = $(FREEIPMI_PLUGIN_FILES) diff --git a/aclk/aclk.c b/aclk/aclk.c index 3ff5515684..b7b6a3d81c 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -489,15 +489,15 @@ static int aclk_get_transport_idx(aclk_env_t *env) { } #endif -ACLK_STATUS aclk_status = ACLK_STATUS_INITIALIZING; +ACLK_STATUS aclk_status = ACLK_STATUS_NONE; const char *aclk_status_to_string(void) { switch(aclk_status) { case ACLK_STATUS_CONNECTED: return "connected"; - case ACLK_STATUS_INITIALIZING: - return "initializing"; + case ACLK_STATUS_NONE: + return "none"; case ACLK_STATUS_DISABLED: return "disabled"; diff --git a/aclk/aclk.h b/aclk/aclk.h index 17353be29d..0badc1a628 100644 --- a/aclk/aclk.h +++ b/aclk/aclk.h @@ -15,7 +15,7 @@ typedef enum __attribute__((packed)) { ACLK_STATUS_CONNECTED = 0, - ACLK_STATUS_INITIALIZING, + ACLK_STATUS_NONE, ACLK_STATUS_DISABLED, ACLK_STATUS_NO_CLOUD_URL, ACLK_STATUS_INVALID_CLOUD_URL, diff --git a/aclk/aclk_capas.c b/aclk/aclk_capas.c index a897d30446..a81116faf6 100644 --- a/aclk/aclk_capas.c +++ b/aclk/aclk_capas.c @@ -4,7 +4,7 @@ #include "ml/ml.h" -#define HTTP_API_V2_VERSION 4 +#define HTTP_API_V2_VERSION 5 const struct capability *aclk_get_agent_capas() { diff --git a/claim/claim.c b/claim/claim.c index 48ffcc4c72..1f61fdee28 100644 --- a/claim/claim.c +++ b/claim/claim.c @@ -42,16 +42,16 @@ char *get_agent_claimid() } #define CLAIMING_COMMAND_LENGTH 16384 -#define CLAIMING_PROXY_LENGTH CLAIMING_COMMAND_LENGTH/4 +#define CLAIMING_PROXY_LENGTH (CLAIMING_COMMAND_LENGTH/4) extern struct registry registry; /* rrd_init() and post_conf_load() must have been called before this function */ -void claim_agent(char *claiming_arguments) +CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg) { - if (!netdata_cloud_enabled) { + if (!force || !netdata_cloud_enabled) { error("Refusing to claim agent -> cloud functionality has been disabled"); - return; + return CLAIM_AGENT_CLOUD_DISABLED; } #ifndef DISABLE_CLOUD @@ -62,8 +62,11 @@ void claim_agent(char *claiming_arguments) // This is guaranteed to be set early in main via post_conf_load() char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL); - if (cloud_base_url == NULL) - fatal("Do not move the cloud base url out of post_conf_load!!"); + if (cloud_base_url == NULL) { + internal_fatal(true, "Do not move the cloud base url out of post_conf_load!!"); + return CLAIM_AGENT_NO_CLOUD_URL; + } + const char *proxy_str; ACLK_PROXY_TYPE proxy_type; char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy"; @@ -76,7 +79,6 @@ void claim_agent(char *claiming_arguments) snprintfz(command_buffer, CLAIMING_COMMAND_LENGTH, "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s", - proxy_flag, netdata_configured_hostname, localhost->machine_guid, @@ -87,7 +89,7 @@ void claim_agent(char *claiming_arguments) fp_child_output = netdata_popen(command_buffer, &command_pid, &fp_child_input); if(!fp_child_output) { error("Cannot popen(\"%s\").", command_buffer); - return; + return CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT; } netdata_log_info("Waiting for claiming command to finish."); while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp_child_output) != NULL) {;} @@ -95,25 +97,31 @@ void claim_agent(char *claiming_arguments) netdata_log_info("Agent claiming command returned with code %d", exit_code); if (0 == exit_code) { load_claiming_state(); - return; + return CLAIM_AGENT_OK; } if (exit_code < 0) { error("Agent claiming command failed to complete its run."); - return; + return CLAIM_AGENT_CLAIM_SCRIPT_FAILED; } errno = 0; unsigned maximum_known_exit_code = sizeof(claiming_errors) / sizeof(claiming_errors[0]) - 1; if ((unsigned)exit_code > maximum_known_exit_code) { error("Agent failed to be claimed with an unknown error."); - return; + return CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE; } + error("Agent failed to be claimed with the following error message:"); error("\"%s\"", claiming_errors[exit_code]); + + if(msg) *msg = claiming_errors[exit_code]; + #else UNUSED(claiming_arguments); UNUSED(claiming_errors); #endif + + return CLAIM_AGENT_FAILED_WITH_MESSAGE; } #ifdef ENABLE_ACLK @@ -181,7 +189,7 @@ void load_claiming_state(void) freez(claimed_id); netdata_log_info("File '%s' was found. Setting state to AGENT_CLAIMED.", filename); - netdata_cloud_enabled = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1); + netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled); #endif } @@ -193,6 +201,10 @@ struct config cloud_config = { .first_section = NULL, void load_cloud_conf(int silent) { + char *nd_disable_cloud = getenv("NETDATA_DISABLE_CLOUD"); + if (nd_disable_cloud && !strncmp(nd_disable_cloud, "1", 1)) + netdata_cloud_enabled = CONFIG_BOOLEAN_NO; + char *filename; errno = 0; @@ -201,8 +213,254 @@ void load_cloud_conf(int silent) filename = strdupz_path_subpath(netdata_configured_varlib_dir, "cloud.d/cloud.conf"); ret = appconfig_load(&cloud_config, filename, 1, NULL); - if(!ret && !silent) { + if(!ret && !silent) netdata_log_info("CONFIG: cannot load cloud config '%s'. Running with internal defaults.", filename); - } + freez(filename); + + // -------------------------------------------------------------------- + // Check if the cloud is enabled + +#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK ) + netdata_cloud_enabled = CONFIG_BOOLEAN_NO; +#else + netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled); +#endif + + // This must be set before any point in the code that accesses it. Do not move it from this function. + appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL); +} + +static char *netdata_random_session_id_filename = NULL; +static uuid_t netdata_random_session_id = { 0 }; + +bool netdata_random_session_id_generate(void) { + static char guid[UUID_STR_LEN] = ""; + + uuid_generate_random(netdata_random_session_id); + uuid_unparse_lower(netdata_random_session_id, guid); + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/netdata_random_session_id", netdata_configured_varlib_dir); + + bool ret = true; + + (void)unlink(filename); + + // save it + int fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 640); + if(fd == -1) { + error("Cannot create random session id file '%s'.", filename); + ret = false; + } + + if(write(fd, guid, UUID_STR_LEN - 1) != UUID_STR_LEN - 1) { + error("Cannot write the random session id file '%s'.", filename); |