From 440bd51e08fdfa2a4daa191fb68643456028a753 Mon Sep 17 00:00:00 2001 From: vkalintiris Date: Wed, 26 Jul 2023 15:30:49 +0300 Subject: Refactor RRD code. (#15423) * Storage engine. * Host indexes to rrdb * Move globals to rrdb * Move storage_tiers_backfill to rrdb * default_rrd_update_every to rrdb * default_rrd_history_entries to rrdb * gap_when_lost_iterations_above to rrdb * rrdset_free_obsolete_time_s to rrdb * libuv_worker_threads to rrdb * ieee754_doubles to rrdb * rrdhost_free_orphan_time_s to rrdb * rrd_rwlock to rrdb * localhost to rrdb * rm extern from func decls * mv rrd macro under rrd.h * default_rrdeng_page_cache_mb to rrdb * default_rrdeng_extent_cache_mb to rrdb * db_engine_journal_check to rrdb * default_rrdeng_disk_quota_mb to rrdb * default_multidb_disk_quota_mb to rrdb * multidb_ctx to rrdb * page_type_size to rrdb * tier_page_size to rrdb * No storage_engine_id in rrdim functions * storage_engine_id is provided by st * Update to fix merge conflict. * Update field name * Remove unnecessary macros from rrd.h * Rm unused type decls * Rm duplicate func decls * make internal function static * Make the rest of public dbengine funcs accept a storage_instance. * No more rrdengine_instance :) * rm rrdset_debug from rrd.h * Use rrdb to access globals in ML and ACLK Missed due to not having the submodules in the worktree. * rm total_number * rm RRDVAR_TYPE_TOTAL * rm unused inline * Rm names from typedef'd enums * rm unused header include * Move include * Rm unused header include * s/rrdhost_find_or_create/rrdhost_get_or_create/g * s/find_host_by_node_id/rrdhost_find_by_node_id/ Also, remove duplicate definition in rrdcontext.c * rm macro used only once * rm macro used only once * Reduce rrd.h api by moving funcs into a collector specific utils header * Remove unused func * Move parser specific function out of rrd.h * return storage_number instead of void pointer * move code related to rrd initialization out of rrdhost.c * Remove tier_grouping from rrdim_tier Saves 8 * storage_tiers bytes per dimension. * Fix rebase * s/rrd_update_every/update_every/ * Mark functions as static and constify args * Add license notes and file to build systems. * Remove remaining non-log/config mentions of memory mode * Move rrdlabels api to separate file. Also, move localhost functions that loads labels outside of database/ and into daemon/ * Remove function decl in rrd.h * merge rrdhost_cache_dir_for_rrdset_alloc into rrdset_cache_dir * Do not expose internal function from rrd.h * Rm NETDATA_RRD_INTERNALS Only one function decl is covered. We have more database internal functions that we currently expose for no good reason. These will be placed in a separate internal header in follow up PRs. * Add license note * Include libnetdata.h instead of aral.h * Use rrdb to access localhost * Fix builds without dbengine * Add header to build system files * Add rrdlabels.h to build systems * Move func def from rrd.h to rrdhost.c * Fix macos build * Rm non-existing function * Rebase master * Define buffer length macro in ad_charts. * Fix FreeBSD builds. * Mark functions static * Rm func decls without definitions * Rebase master * Rebase master * Properly initialize value of storage tiers. * Fix build after rebase. --- CMakeLists.txt | 4 + Makefile.am | 4 + aclk/aclk.c | 36 +- aclk/aclk_capas.c | 6 +- aclk/aclk_proxy.c | 2 + aclk/aclk_proxy.h | 2 + aclk/aclk_query.c | 2 +- aclk/aclk_rrdhost_state.h | 2 + aclk/aclk_rx_msgs.c | 8 +- aclk/aclk_stats.c | 50 +- aclk/aclk_tx_msgs.c | 30 +- aclk/aclk_util.c | 14 +- aclk/schema-wrappers/node_info.cc | 2 + aclk/schema-wrappers/proto_2_json.cc | 2 + claim/claim.c | 13 +- collectors/cgroups.plugin/sys_fs_cgroup.c | 6 +- collectors/cgroups.plugin/tests/test_doubles.c | 8 +- collectors/diskspace.plugin/plugin_diskspace.c | 9 +- collectors/ebpf.plugin/ebpf_unittest.h | 4 +- collectors/freebsd.plugin/plugin_freebsd.c | 4 +- collectors/freebsd.plugin/plugin_freebsd.h | 1 + collectors/idlejitter.plugin/plugin_idlejitter.c | 4 +- collectors/macos.plugin/plugin_macos.c | 4 +- collectors/macos.plugin/plugin_macos.h | 1 + collectors/plugins.d/local_listeners.c | 2 + collectors/plugins.d/plugins_d.c | 4 +- collectors/plugins.d/pluginsd_parser.c | 41 +- collectors/proc.plugin/ipc.c | 12 +- collectors/proc.plugin/plugin_proc.c | 4 +- collectors/proc.plugin/proc_mdstat.c | 1 + collectors/proc.plugin/proc_net_dev.c | 4 +- collectors/proc.plugin/proc_net_netstat.c | 4 +- collectors/proc.plugin/proc_net_sockstat.c | 16 +- collectors/proc.plugin/proc_net_softnet_stat.c | 1 + collectors/proc.plugin/proc_net_stat_conntrack.c | 4 +- collectors/proc.plugin/proc_net_wireless.c | 2 + collectors/proc.plugin/proc_stat.c | 4 +- collectors/proc.plugin/sys_devices_pci_aer.c | 2 +- collectors/profile.plugin/plugin_profile.cc | 4 +- collectors/statsd.plugin/statsd.c | 25 +- collectors/tc.plugin/plugin_tc.c | 14 +- collectors/timex.plugin/plugin_timex.c | 4 +- collectors/utils.h | 42 ++ daemon/analytics.c | 44 +- daemon/analytics.h | 2 +- daemon/buildinfo.c | 4 +- daemon/commands.c | 6 +- daemon/global_statistics.c | 182 +++--- daemon/main.c | 330 +++++++--- daemon/main.h | 2 + daemon/service.c | 22 +- daemon/unit_test.c | 62 +- database/contexts/api_v2.c | 29 +- database/contexts/internal.h | 1 - database/contexts/query_scope.c | 2 +- database/contexts/query_target.c | 38 +- database/contexts/rrdcontext.c | 25 +- database/contexts/rrdcontext.h | 2 +- database/contexts/worker.c | 12 +- database/engine/cache.c | 2 + database/engine/cache.h | 2 + database/engine/datafile.c | 4 +- database/engine/journalfile.c | 12 +- database/engine/metric.c | 2 + database/engine/metric.h | 2 + database/engine/pagecache.c | 9 +- database/engine/pdc.c | 4 +- database/engine/rrddim_eng.h | 73 +++ database/engine/rrdengine.c | 25 +- database/engine/rrdengine.h | 2 - database/engine/rrdengineapi.c | 155 +++-- database/engine/rrdengineapi.h | 63 +- database/engine/rrdenginelib.c | 4 +- database/ram/rrddim_mem.c | 25 +- database/ram/rrddim_mem.h | 37 +- database/rrd.c | 441 ++++++++++--- database/rrd.h | 681 ++++---------------- database/rrdcalc.c | 2 +- database/rrddim.c | 443 ++++++------- database/rrdfamily.c | 1 - database/rrdfunctions.c | 7 +- database/rrdfunctions.h | 2 + database/rrdhost.c | 702 ++++----------------- database/rrdlabels.c | 32 +- database/rrdlabels.h | 60 ++ database/rrdset.c | 659 +++++++++---------- database/rrdvar.c | 5 - database/rrdvar.h | 3 +- database/sqlite/sqlite_aclk.c | 16 +- database/sqlite/sqlite_aclk.h | 2 +- database/sqlite/sqlite_aclk_alert.c | 16 +- database/sqlite/sqlite_aclk_node.c | 12 +- database/sqlite/sqlite_functions.c | 16 +- database/sqlite/sqlite_functions.h | 2 +- database/sqlite/sqlite_health.c | 14 +- database/sqlite/sqlite_metadata.c | 34 +- database/storage_engine.c | 143 ++--- database/storage_engine.h | 451 ++++++++++++- database/storage_engine_types.h | 56 ++ exporting/check_filters.c | 6 +- exporting/exporting_engine.c | 2 +- exporting/graphite/graphite.c | 4 +- exporting/json/json.c | 4 +- exporting/opentsdb/opentsdb.c | 8 +- exporting/process_data.c | 8 +- exporting/prometheus/prometheus.c | 6 +- exporting/prometheus/remote_write/remote_write.c | 12 +- exporting/read_config.c | 4 +- exporting/send_internal_metrics.c | 2 +- exporting/tests/exporting_fixtures.c | 4 +- exporting/tests/netdata_doubles.c | 12 +- exporting/tests/test_exporting_engine.c | 4 +- health/health.c | 16 +- health/health.h | 7 - health/health_config.c | 2 +- libnetdata/aral/aral.c | 3 +- libnetdata/aral/aral.h | 1 + libnetdata/circular_buffer/circular_buffer.c | 2 + libnetdata/circular_buffer/circular_buffer.h | 2 + libnetdata/json/jsmn.c | 2 + libnetdata/json/jsmn.h | 2 + libnetdata/libnetdata.c | 4 +- libnetdata/libnetdata.h | 3 +- libnetdata/onewayalloc/onewayalloc.c | 2 + libnetdata/onewayalloc/onewayalloc.h | 2 + libnetdata/socket/security.c | 2 + libnetdata/socket/security.h | 2 + libnetdata/string/string.h | 1 + libnetdata/worker_utilization/worker_utilization.c | 2 + libnetdata/worker_utilization/worker_utilization.h | 2 + ml/ad_charts.cc | 113 ++-- ml/ml-dummy.c | 2 + ml/ml.cc | 2 +- registry/registry.c | 14 +- streaming/compression.c | 2 + streaming/receiver.c | 53 +- streaming/replication.c | 18 +- streaming/rrdpush.c | 8 +- streaming/rrdpush.h | 4 +- streaming/sender.c | 2 +- tests/ebpf/sync_tester.c | 2 + web/api/badges/web_buffer_svg.c | 6 +- web/api/formatters/charts2json.c | 4 +- web/api/formatters/json_wrapper.c | 10 +- web/api/formatters/rrd2json.c | 3 +- web/api/formatters/value/value.c | 2 +- web/api/health/health_cmdapi.c | 4 +- web/api/queries/query.c | 69 +- web/api/queries/weights.c | 8 +- web/api/web_api.c | 2 +- web/api/web_api_v1.c | 92 +-- web/api/web_api_v2.c | 10 +- web/server/h2o/http_server.c | 4 +- web/server/static/static-threaded.c | 2 +- web/server/web_client.c | 20 +- 155 files changed, 3063 insertions(+), 2918 deletions(-) create mode 100644 collectors/utils.h create mode 100644 database/engine/rrddim_eng.h mode change 100755 => 100644 database/engine/rrdengineapi.c create mode 100644 database/rrdlabels.h create mode 100644 database/storage_engine_types.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cb79c54a6c..e3fffbd045 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -733,6 +733,7 @@ set(RRD_PLUGIN_FILES database/rrdfunctions.h database/rrdhost.c database/rrdlabels.c + database/rrdlabels.h database/rrd.c database/rrd.h database/rrdset.c @@ -742,6 +743,7 @@ set(RRD_PLUGIN_FILES database/rrdvar.h database/storage_engine.c database/storage_engine.h + database/storage_engine_types.h database/ram/rrddim_mem.c database/ram/rrddim_mem.h database/sqlite/sqlite_metadata.c @@ -773,6 +775,7 @@ set(RRD_PLUGIN_FILES database/engine/rrdenginelib.h database/engine/rrdengineapi.c database/engine/rrdengineapi.h + database/engine/rrddim_eng.h database/engine/pagecache.c database/engine/pagecache.h database/engine/cache.c @@ -1055,6 +1058,7 @@ ENDIF() set(NETDATA_FILES collectors/all.h + collectors/utils.h ${DAEMON_FILES} ${API_PLUGIN_FILES} ${EXPORTING_ENGINE_FILES} diff --git a/Makefile.am b/Makefile.am index c1017b289e..72faf8ce32 100644 --- a/Makefile.am +++ b/Makefile.am @@ -462,6 +462,7 @@ RRD_PLUGIN_FILES = \ database/rrdfamily.c \ database/rrdhost.c \ database/rrdlabels.c \ + database/rrdlabels.h \ database/rrd.c \ database/rrd.h \ database/rrdset.c \ @@ -473,6 +474,7 @@ RRD_PLUGIN_FILES = \ database/rrdvar.h \ database/storage_engine.c \ database/storage_engine.h \ + database/storage_engine_types.h \ database/ram/rrddim_mem.c \ database/ram/rrddim_mem.h \ database/sqlite/sqlite_functions.c \ @@ -545,6 +547,7 @@ libjudy_a-JudyLTables.$(OBJEXT) : CFLAGS += -I$(abs_top_srcdir)/libnetdata/libju if ENABLE_DBENGINE RRD_PLUGIN_FILES += \ + database/engine/rrddim_eng.h \ database/engine/rrdengine.c \ database/engine/rrdengine.h \ database/engine/rrddiskprotocol.h \ @@ -1070,6 +1073,7 @@ endif #ENABLE_H2O NETDATA_FILES = \ collectors/all.h \ + collectors/utils.h \ $(DAEMON_FILES) \ $(LIBNETDATA_FILES) \ $(API_PLUGIN_FILES) \ diff --git a/aclk/aclk.c b/aclk/aclk.c index 312db076ff..59c3106dfa 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -913,15 +913,15 @@ void aclk_host_state_update(RRDHOST *host, int cmd) // node_id not found aclk_query_t create_query; create_query = aclk_query_new(REGISTER_NODE); - rrdhost_aclk_state_lock(localhost); + rrdhost_aclk_state_lock(rrdb.localhost); node_instance_creation_t node_instance_creation = { - .claim_id = localhost->aclk_state.claimed_id, + .claim_id = rrdb.localhost->aclk_state.claimed_id, .hops = host->system_info->hops, .hostname = rrdhost_hostname(host), .machine_guid = host->machine_guid}; create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE; create_query->data.bin_payload.msg_name = "CreateNodeInstance"; netdata_log_info("Registering host=%s, hops=%u", host->machine_guid, host->system_info->hops); @@ -942,10 +942,10 @@ void aclk_host_state_update(RRDHOST *host, int cmd) node_state_update.capabilities = aclk_get_agent_capas(); - rrdhost_aclk_state_lock(localhost); - node_state_update.claim_id = localhost->aclk_state.claimed_id; + rrdhost_aclk_state_lock(rrdb.localhost); + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id; query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); netdata_log_info("Queuing status update for node=%s, live=%d, hops=%u",(char*)node_state_update.node_id, cmd, host->system_info->hops); @@ -986,10 +986,10 @@ void aclk_send_node_instances() } node_state_update.capabilities = aclk_get_node_instance_capas(host); - rrdhost_aclk_state_lock(localhost); - node_state_update.claim_id = localhost->aclk_state.claimed_id; + rrdhost_aclk_state_lock(rrdb.localhost); + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id; query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); netdata_log_info("Queuing status update for node=%s, live=%d, hops=%d",(char*)node_state_update.node_id, list->live, list->hops); @@ -1010,10 +1010,10 @@ void aclk_send_node_instances() uuid_unparse_lower(list->host_id, (char*)node_instance_creation.machine_guid); create_query->data.bin_payload.topic = ACLK_TOPICID_CREATE_NODE; create_query->data.bin_payload.msg_name = "CreateNodeInstance"; - rrdhost_aclk_state_lock(localhost); - node_instance_creation.claim_id = localhost->aclk_state.claimed_id, + rrdhost_aclk_state_lock(rrdb.localhost); + node_instance_creation.claim_id = rrdb.localhost->aclk_state.claimed_id, create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); netdata_log_info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid, list->hops); freez((void *)node_instance_creation.machine_guid); @@ -1124,9 +1124,9 @@ char *aclk_state(void) buffer_sprintf(wb, "\n\tNode ID: %s\n", node_id); } - buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == localhost ? "self" : "child"); + buffer_sprintf(wb, "\tStreaming Hops: %d\n\tRelationship: %s", host->system_info->hops, host == rrdb.localhost ? "self" : "child"); - if (host != localhost) + if (host != rrdb.localhost) buffer_sprintf(wb, "\n\tStreaming Connection Live: %s", host->receiver ? "true" : "false"); buffer_strcat(wb, "\n\tAlert Streaming Status:"); @@ -1270,10 +1270,10 @@ char *aclk_state_json(void) tmp = json_object_new_int(host->system_info->hops); json_object_object_add(nodeinstance, "streaming-hops", tmp); - tmp = json_object_new_string(host == localhost ? "self" : "child"); + tmp = json_object_new_string(host == rrdb.localhost ? "self" : "child"); json_object_object_add(nodeinstance, "relationship", tmp); - tmp = json_object_new_boolean((host->receiver || host == localhost)); + tmp = json_object_new_boolean((host->receiver || host == rrdb.localhost)); json_object_object_add(nodeinstance, "streaming-online", tmp); tmp = json_object_new_object(); @@ -1292,7 +1292,7 @@ char *aclk_state_json(void) } void add_aclk_host_labels(void) { - DICTIONARY *labels = localhost->rrdlabels; + DICTIONARY *labels = rrdb.localhost->rrdlabels; #ifdef ENABLE_ACLK rrdlabels_add(labels, "_aclk_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK); @@ -1324,5 +1324,5 @@ void aclk_queue_node_info(RRDHOST *host, bool immediate) { struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config; if (likely(wc)) - wc->node_info_send_time = (host == localhost || immediate) ? 1 : now_realtime_sec(); + wc->node_info_send_time = (host == rrdb.localhost || immediate) ? 1 : now_realtime_sec(); } diff --git a/aclk/aclk_capas.c b/aclk/aclk_capas.c index e8d85a2b9f..28c8825bce 100644 --- a/aclk/aclk_capas.c +++ b/aclk/aclk_capas.c @@ -21,19 +21,19 @@ const struct capability *aclk_get_agent_capas() { .name = NULL, .version = 0, .enabled = 0 } }; agent_capabilities[2].version = ml_capable() ? 1 : 0; - agent_capabilities[2].enabled = ml_enabled(localhost); + agent_capabilities[2].enabled = ml_enabled(rrdb.localhost); agent_capabilities[3].version = enable_metric_correlations ? metric_correlations_version : 0; agent_capabilities[3].enabled = enable_metric_correlations; - agent_capabilities[7].enabled = localhost->health.health_enabled; + agent_capabilities[7].enabled = rrdb.localhost->health.health_enabled; return agent_capabilities; } struct capability *aclk_get_node_instance_capas(RRDHOST *host) { - bool functions = (host == localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS))); + bool functions = (host == rrdb.localhost || (host->receiver && stream_has_capability(host->receiver, STREAM_CAP_FUNCTIONS))); struct capability ni_caps[] = { { .name = "proto", .version = 1, .enabled = 1 }, diff --git a/aclk/aclk_proxy.c b/aclk/aclk_proxy.c index 4af46208f7..dd79f9061f 100644 --- a/aclk/aclk_proxy.c +++ b/aclk/aclk_proxy.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #include "aclk_proxy.h" #include "daemon/common.h" diff --git a/aclk/aclk_proxy.h b/aclk/aclk_proxy.h index b4ceb7df84..e8fe2ae176 100644 --- a/aclk/aclk_proxy.h +++ b/aclk/aclk_proxy.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #ifndef ACLK_PROXY_H #define ACLK_PROXY_H diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 07d571be17..697e76118b 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -128,7 +128,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) ACLK_STATS_UNLOCK; } - w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path); + w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path); web_client_timeout_checkpoint_response_ready(w, &t); if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) { diff --git a/aclk/aclk_rrdhost_state.h b/aclk/aclk_rrdhost_state.h index 5c8a2ddc94..ae7f548c90 100644 --- a/aclk/aclk_rrdhost_state.h +++ b/aclk/aclk_rrdhost_state.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #ifndef ACLK_RRDHOST_STATE_H #define ACLK_RRDHOST_STATE_H diff --git a/aclk/aclk_rx_msgs.c b/aclk/aclk_rx_msgs.c index 84ade2b346..463db0e94a 100644 --- a/aclk/aclk_rx_msgs.c +++ b/aclk/aclk_rx_msgs.c @@ -279,7 +279,7 @@ int create_node_instance_result(const char *msg, size_t msg_len) RRDHOST *host = rrdhost_find_by_guid(res.machine_guid); if (likely(host)) { - if (host == localhost) { + if (host == rrdb.localhost) { node_state_update.live = 1; node_state_update.hops = 0; } else { @@ -289,10 +289,10 @@ int create_node_instance_result(const char *msg, size_t msg_len) node_state_update.capabilities = aclk_get_node_instance_capas(host); } - rrdhost_aclk_state_lock(localhost); - node_state_update.claim_id = localhost->aclk_state.claimed_id; + rrdhost_aclk_state_lock(rrdb.localhost); + node_state_update.claim_id = rrdb.localhost->aclk_state.claimed_id; query->data.bin_payload.payload = generate_node_instance_connection(&query->data.bin_payload.size, &node_state_update); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); freez((void *)node_state_update.capabilities); diff --git a/aclk/aclk_stats.c b/aclk/aclk_stats.c index f4672882b7..03e9d95e25 100644 --- a/aclk/aclk_stats.c +++ b/aclk/aclk_stats.c @@ -38,7 +38,7 @@ static void aclk_stats_collect(struct aclk_metrics_per_sample *per_sample, struc if (unlikely(!st_aclkstats)) { st_aclkstats = rrdset_create_localhost( "netdata", "aclk_status", NULL, "aclk", NULL, "ACLK/Cloud connection status", - "connected", "netdata", "stats", 200000, localhost->rrd_update_every, RRDSET_TYPE_LINE); + "connected", "netdata", "stats", 200000, rrdb.localhost->update_every, RRDSET_TYPE_LINE); rd_online_status = rrddim_add(st_aclkstats, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -57,10 +57,10 @@ static void aclk_stats_query_queue(struct aclk_metrics_per_sample *per_sample) if (unlikely(!st_query_thread)) { st_query_thread = rrdset_create_localhost( "netdata", "aclk_query_per_second", NULL, "aclk", NULL, "ACLK Queries per second", "queries/s", - "netdata", "stats", 200001, localhost->rrd_update_every, RRDSET_TYPE_AREA); + "netdata", "stats", 200001, rrdb.localhost->update_every, RRDSET_TYPE_AREA); - rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); - rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); + rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } rrddim_set_by_pointer(st_query_thread, rd_queued, per_sample->queries_queued); @@ -79,7 +79,7 @@ static void aclk_stats_latency(struct aclk_metrics_per_sample *per_sample) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_latency_mqtt", NULL, "aclk", NULL, "ACLK Message Publish Latency", "ms", - "netdata", "stats", 200002, localhost->rrd_update_every, RRDSET_TYPE_LINE); + "netdata", "stats", 200002, rrdb.localhost->update_every, RRDSET_TYPE_LINE); rd_avg = rrddim_add(st, "avg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -105,10 +105,10 @@ static void aclk_stats_cloud_req(struct aclk_metrics_per_sample *per_sample) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_cloud_req", NULL, "aclk", NULL, "Requests received from cloud", "req/s", - "netdata", "stats", 200005, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200005, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); - rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); - rd_rq_err = rrddim_add(st, "malformed", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_err = rrddim_add(st, "malformed", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } rrddim_set_by_pointer(st, rd_rq_rcvd, per_sample->cloud_req_recvd - per_sample->cloud_req_err); @@ -125,10 +125,10 @@ static void aclk_stats_cloud_req_type(struct aclk_metrics_per_sample *per_sample if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_processed_query_type", NULL, "aclk", NULL, "Query thread commands processed by their type", "cmd/s", - "netdata", "stats", 200006, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200006, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); for (int i = 0; i < ACLK_QUERY_TYPE_COUNT; i++) - dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + dims[i] = rrddim_add(st, aclk_query_get_name(i, 1), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } @@ -167,10 +167,10 @@ static void aclk_stats_cloud_req_http_type(struct aclk_metrics_per_sample *per_s if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_cloud_req_http_type", NULL, "aclk", NULL, "Requests received from cloud via HTTP by their type", "req/s", - "netdata", "stats", 200007, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200007, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++) - rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_types[i] = rrddim_add(st, cloud_req_http_type_names[i], NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } for (int i = 0; i < ACLK_STATS_CLOUD_HTTP_REQ_TYPE_CNT; i++) @@ -189,12 +189,12 @@ static void aclk_stats_query_threads(uint32_t *queries_per_thread) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_query_threads", NULL, "aclk", NULL, "Queries Processed Per Thread", "req/s", - "netdata", "stats", 200009, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200009, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); for (int i = 0; i < aclk_stats_cfg.query_thread_count; i++) { if (snprintfz(dim_name, MAX_DIM_NAME, "Query %d", i) < 0) netdata_log_error("snprintf encoding error"); - aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } } @@ -215,11 +215,11 @@ static void aclk_stats_query_time(struct aclk_metrics_per_sample *per_sample) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_query_time", NULL, "aclk", NULL, "Time it took to process cloud requested DB queries", "us", - "netdata", "stats", 200008, localhost->rrd_update_every, RRDSET_TYPE_LINE); + "netdata", "stats", 200008, rrdb.localhost->update_every, RRDSET_TYPE_LINE); - rd_rq_avg = rrddim_add(st, "avg", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); - rd_rq_max = rrddim_add(st, "max", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); - rd_rq_total = rrddim_add(st, "total", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_avg = rrddim_add(st, "avg", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_max = rrddim_add(st, "max", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); + rd_rq_total = rrddim_add(st, "total", NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } if(per_sample->cloud_q_process_count) @@ -240,10 +240,10 @@ static void aclk_stats_newproto_rx(uint32_t *rx_msgs_sample) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_protobuf_rx_types", NULL, "aclk", NULL, "Received new cloud architecture messages by their type.", "msg/s", - "netdata", "stats", 200010, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200010, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); for (unsigned int i = 0; i < aclk_stats_cfg.proto_hdl_cnt; i++) { - aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE); + aclk_stats_cfg.rx_msg_dims[i] = rrddim_add(st, rx_handler_get_name(i), NULL, 1, rrdb.localhost->update_every, RRD_ALGORITHM_ABSOLUTE); } } @@ -284,7 +284,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats) if (unlikely(!st)) { st = rrdset_create_localhost( "netdata", "aclk_openssl_bytes", NULL, "aclk", NULL, "Received and Sent bytes.", "B/s", - "netdata", "stats", 200011, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200011, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); rd_sent = rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); rd_recvd = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -293,7 +293,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats) if (unlikely(!st_txbuf_perc)) { st_txbuf_perc = rrdset_create_localhost( "netdata", "aclk_mqtt_tx_perc", NULL, "aclk", NULL, "Actively used percentage of MQTT Tx Buffer,", "%", - "netdata", "stats", 200012, localhost->rrd_update_every, RRDSET_TYPE_LINE); + "netdata", "stats", 200012, rrdb.localhost->update_every, RRDSET_TYPE_LINE); rd_txbuf_perc = rrddim_add(st_txbuf_perc, "used", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE); } @@ -301,7 +301,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats) if (unlikely(!st_txbuf)) { st_txbuf = rrdset_create_localhost( "netdata", "aclk_mqtt_tx_queue", NULL, "aclk", NULL, "State of transmit MQTT queue.", "B", - "netdata", "stats", 200013, localhost->rrd_update_every, RRDSET_TYPE_LINE); + "netdata", "stats", 200013, rrdb.localhost->update_every, RRDSET_TYPE_LINE); rd_tx_buffer_usable = rrddim_add(st_txbuf, "usable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rd_tx_buffer_reclaimable = rrddim_add(st_txbuf, "reclaimable", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -313,7 +313,7 @@ static void aclk_stats_mqtt_wss(struct mqtt_wss_stats *stats) if (unlikely(!st_timing)) { st_timing = rrdset_create_localhost( "netdata", "aclk_mqtt_wss_time", NULL, "aclk", NULL, "Time spent handling MQTT, WSS, SSL and network communication.", "us", - "netdata", "stats", 200014, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + "netdata", "stats", 200014, rrdb.localhost->update_every, RRDSET_TYPE_STACKED); rd_keepalive = rrddim_add(st_timing, "keep-alive", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rd_read_socket = rrddim_add(st_timing, "socket_read_ssl", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -379,7 +379,7 @@ void *aclk_stats_main_thread(void *ptr) heartbeat_t hb; heartbeat_init(&hb); - usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC; + usec_t step_ut = rrdb.localhost->update_every * USEC_PER_SEC; struct aclk_metrics_per_sample per_sample; struct aclk_metrics permanent; diff --git a/aclk/aclk_tx_msgs.c b/aclk/aclk_tx_msgs.c index 26e20dfb2c..03440aeeb0 100644 --- a/aclk/aclk_tx_msgs.c +++ b/aclk/aclk_tx_msgs.c @@ -218,19 +218,19 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable .capabilities = aclk_get_agent_capas() }; - rrdhost_aclk_state_lock(localhost); - if (unlikely(!localhost->aclk_state.claimed_id)) { + rrdhost_aclk_state_lock(rrdb.localhost); + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) { netdata_log_error("Internal error. Should not come here if not claimed"); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); return 0; } - if (localhost->aclk_state.prev_claimed_id) - conn.claim_id = localhost->aclk_state.prev_claimed_id; + if (rrdb.localhost->aclk_state.prev_claimed_id) + conn.claim_id = rrdb.localhost->aclk_state.prev_claimed_id; else - conn.claim_id = localhost->aclk_state.claimed_id; + conn.claim_id = rrdb.localhost->aclk_state.claimed_id; char *msg = generate_update_agent_connection(&len, &conn); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); if (!msg) { netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload"); @@ -238,9 +238,9 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable } pid = aclk_send_bin_message_subtopic_pid(client, msg, len, ACLK_TOPICID_AGENT_CONN, "UpdateAgentConnection"); - if (localhost->aclk_state.prev_claimed_id) { - freez(localhost->aclk_state.prev_claimed_id); - localhost->aclk_state.prev_claimed_id = NULL; + if (rrdb.localhost->aclk_state.prev_claimed_id) { + freez(rrdb.localhost->aclk_state.prev_claimed_id); + rrdb.localhost->aclk_state.prev_claimed_id = NULL; } return pid; } @@ -253,16 +253,16 @@ char *aclk_generate_lwt(size_t *size) { .capabilities = NULL }; - rrdhost_aclk_state_lock(localhost); - if (unlikely(!localhost->aclk_state.claimed_id)) { + rrdhost_aclk_state_lock(rrdb.localhost); + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) { netdata_log_error("Internal error. Should not come here if not claimed"); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); return NULL; } - conn.claim_id = localhost->aclk_state.claimed_id; + conn.claim_id = rrdb.localhost->aclk_state.claimed_id; char *msg = generate_update_agent_connection(size, &conn); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); if (!msg) netdata_log_error("Error generating agent::v1::UpdateAgentConnection payload for LWT"); diff --git a/aclk/aclk_util.c b/aclk/aclk_util.c index 00920e0690..cf5ee7add3 100644 --- a/aclk/aclk_util.c +++ b/aclk/aclk_util.c @@ -183,20 +183,20 @@ static void topic_generate_final(struct aclk_topic *t) { if (!replace_tag) return; - rrdhost_aclk_state_lock(localhost); - if (unlikely(!localhost->aclk_state.claimed_id)) { + rrdhost_aclk_state_lock(rrdb.localhost); + if (unlikely(!rrdb.localhost->aclk_state.claimed_id)) { netdata_log_error("This should never be called if agent not claimed"); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_unlock(rrdb.localhost); return; } - t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(localhost->aclk_state.claimed_id)); + t->topic = mallocz(strlen(t->topic_recvd) + 1 - strlen(CLAIM_ID_REPLACE_TAG) + strlen(rrdb.localhost->aclk_state.claimed_id)); memcpy(t->topic, t->topic_recvd, replace_tag - t->topic_recvd); dest = t->topic + (replace_tag - t->topic_recvd); - memcpy(dest, localhost->aclk_state.claimed_id, strlen(localhost->aclk_state.claimed_id)); - dest += strlen(localhost->aclk_state.claimed_id); - rrdhost_aclk_state_unlock(localhost); + memcpy(dest, rrdb.localhost->aclk_state.claimed_id, strlen(rrdb.localhost->aclk_state.claimed_id)); + dest += strlen(rrdb.localhost->aclk_state.claimed_id); + rrdhost_aclk_state_unlock(rrdb.localhost); replace_tag += strlen(CLAIM_ID_REPLACE_TAG); strcpy(dest, replace_tag); dest += strlen(replace_tag); diff --git a/aclk/schema-wrappers/node_info.cc b/aclk/schema-wrappers/node_info.cc index 5e321f6886..03f423bb93 100644 --- a/aclk/schema-wrappers/node_info.cc +++ b/aclk/schema-wrappers/node_info.cc @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #include "node_info.h" #include "proto/nodeinstance/info/v1/info.pb.h" diff --git a/aclk/schema-wrappers/proto_2_json.cc b/aclk/schema-wrappers/proto_2_json.cc index 8543965107..c67308fd24 100644 --- a/aclk/schema-wrappers/proto_2_json.cc +++ b/aclk/schema-wrappers/proto_2_json.cc @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #include #include diff --git a/claim/claim.c b/claim/claim.c index a441cda4c6..942dd6d73c 100644 --- a/claim/claim.c +++ b/claim/claim.c @@ -35,9 +35,9 @@ static char *claiming_errors[] = { char *get_agent_claimid() { char *result; - rrdhost_aclk_state_lock(localhost); - result = (localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(localhost->aclk_state.claimed_id); - rrdhost_aclk_state_unlock(localhost); + rrdhost_aclk_state_lock(rrdb.localhost); + result = (rrdb.localhost->aclk_state.claimed_id == NULL) ? NULL : strdupz(rrdb.localhost->aclk_state.claimed_id); + rrdhost_aclk_state_unlock(rrdb.localhost); return result; } @@ -81,7 +81,7 @@ CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, con "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s", proxy_flag, netdata_configured_hostname, - localhost->machine_guid, + rrdb.localhost->machine_guid, cloud_base_url, claiming_arguments); @@ -142,6 +142,7 @@ void load_claiming_state(void) #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK ) netdata_cloud_enabled = false; #else + RRDHOST *localhost = rrdb.localhost; uuid_t uuid; // Propagate into aclk and registry. Be kind of atomic... @@ -312,7 +313,7 @@ void claim_reload_all(void) { error_log_limit_unlimited(); load_claiming_state(); registry_update_cloud_base_url(); - rrdpush_send_claimed_id(localhost); + rrdpush_send_claimed_id(rrdb.localhost); error_log_limit_reset(); } @@ -409,7 +410,7 @@ int api_v2_claim(struct web_client *w, char *url) { int ms = 0; do { status = cloud_status(); - if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&localhost->node_id, __ATOMIC_RELAXED)) + if (status == CLOUD_STATUS_ONLINE && __atomic_load_n(&rrdb.localhost->node_id, __ATOMIC_RELAXED)) break; sleep_usec(50 * USEC_PER_MS); diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c index b7a6a46485..c22d8526dc 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -301,9 +301,9 @@ void read_cgroup_plugin_configuration() { throttled_time_hash = simple_hash("throttled_time"); throttled_usec_hash = simple_hash("throttled_usec"); - cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every); - if(cgroup_update_every < localhost->rrd_update_every) - cgroup_update_every = localhost->rrd_update_every; + cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", rrdb.localhost->update_every); + if(cgroup_update_every < rrdb.localhost->update_every) + cgroup_update_every = rrdb.localhost->update_every; cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", (long long)cgroup_check_for_new_every * (long long)cgroup_update_every); if(cgroup_check_for_new_every < cgroup_update_every) diff --git a/collectors/cgroups.plugin/tests/test_doubles.c b/collectors/cgroups.plugin/tests/test_doubles.c index 498f649f5d..3f2df44eaa 100644 --- a/collectors/cgroups.plugin/tests/test_doubles.c +++ b/collectors/cgroups.plugin/tests/test_doubles.c @@ -47,7 +47,7 @@ void mountinfo_free_all(struct mountinfo *mi) RRDSET *rrdset_create_custom( RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every, - RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries) + RRDSET_TYPE chart_type, STORAGE_ENGINE_ID storage_engine_id, long history_entries) { UNUSED(host); UNUSED(type); @@ -62,7 +62,7 @@ RRDSET *rrdset_create_custom( UNUSED(priority); UNUSED(update_every); UNUSED(chart_type); - UNUSED(memory_mode); + UNUSED(storage_engine_id); UNUSED(history_entries); return NULL; @@ -70,7 +70,7 @@ RRDSET *rrdset_create_custom( RRDDIM *rrddim_add_custom( RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, - RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) + RRD_ALGORITHM algorithm, STORAGE_ENGINE_ID storage_engine_id) { UNUSED(st); UNUSED(id); @@ -78,7 +78,7 @@ RRDDIM *rrddim_add_custom( UNUSED(multiplier); UNUSED(divisor); UNUSED(algorithm); - UNUSED(memory_mode); + UNUSED(storage_engine_id); return NULL; } diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index 2153494d9a..7e742f9995 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "../proc.plugin/plugin_proc.h" +#include "collectors/proc.plugin/plugin_proc.h" +#include "collectors/utils.h" #define PLUGIN_DISKSPACE_NAME "diskspace.plugin" @@ -626,9 +627,9 @@ void *diskspace_main(void *ptr) { cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points); - int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", localhost->rrd_update_every); - if(update_every < localhost->rrd_update_every) - update_every = localhost->rrd_update_every; + int update_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "update every", rrdb.localhost->update_every); + if(update_every < rrdb.localhost->update_every) + update_every = rrdb.localhost->update_every; check_for_new_mountpoints_every = (int)config_get_number(CONFIG_SECTION_DISKSPACE, "check for new mount points every", check_for_new_mountpoints_every); if(check_for_new_mountpoints_every < update_every) diff --git a/collectors/ebpf.plugin/ebpf_unittest.h b/collectors/ebpf.plugin/ebpf_unittest.h index 429cbe6288..3defff2284 100644 --- a/collectors/ebpf.plugin/ebpf_unittest.h +++ b/collectors/ebpf.plugin/ebpf_unittest.h @@ -1,5 +1,7 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #ifndef NETDATA_EBPF_PLUGIN_UNITTEST_H_ -# define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1 +#define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1 #include "ebpf.h" diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c index 976fe26fbe..26dab93e50 100644 --- a/collectors/freebsd.plugin/plugin_freebsd.c +++ b/collectors/freebsd.plugin/plugin_freebsd.c @@ -104,7 +104,7 @@ void *freebsd_main(void *ptr) worker_register_job_name(i, freebsd_modules[i].dim); } - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); @@ -124,7 +124,7 @@ void *freebsd_main(void *ptr) netdata_log_debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name); worker_is_busy(i); - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt); if (unlikely(netdata_exit)) break; diff --git a/collectors/freebsd.plugin/plugin_freebsd.h b/collectors/freebsd.plugin/plugin_freebsd.h index af7d0822e0..3756a2f080 100644 --- a/collectors/freebsd.plugin/plugin_freebsd.h +++ b/collectors/freebsd.plugin/plugin_freebsd.h @@ -4,6 +4,7 @@ #define NETDATA_PLUGIN_FREEBSD_H 1 #include "daemon/common.h" +#include "collectors/utils.h" #include diff --git a/collectors/idlejitter.plugin/plugin_idlejitter.c b/collectors/idlejitter.plugin/plugin_idlejitter.c index d90548869f..093fe443c9 100644 --- a/collectors/idlejitter.plugin/plugin_idlejitter.c +++ b/collectors/idlejitter.plugin/plugin_idlejitter.c @@ -38,14 +38,14 @@ void *cpuidlejitter_main(void *ptr) { , "idlejitter.plugin" , NULL , NETDATA_CHART_PRIO_SYSTEM_IDLEJITTER - , localhost->rrd_update_every + , rrdb.localhost->update_every , RRDSET_TYPE_AREA ); RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC; + usec_t update_every_ut = rrdb.localhost->update_every * USEC_PER_SEC; struct timeval before, after; while (service_running(SERVICE_COLLECTORS)) { diff --git a/collectors/macos.plugin/plugin_macos.c b/collectors/macos.plugin/plugin_macos.c index 3aaa46c729..0b60b060a1 100644 --- a/collectors/macos.plugin/plugin_macos.c +++ b/collectors/macos.plugin/plugin_macos.c @@ -53,7 +53,7 @@ void *macos_main(void *ptr) worker_register_job_name(i, macos_modules[i].dim); } - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); @@ -69,7 +69,7 @@ void *macos_main(void *ptr) netdata_log_debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name); worker_is_busy(i); - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt); if (unlikely(netdata_exit)) break; diff --git a/collectors/macos.plugin/plugin_macos.h b/collectors/macos.plugin/plugin_macos.h index 2c673a2243..febdc767e1 100644 --- a/collectors/macos.plugin/plugin_macos.h +++ b/collectors/macos.plugin/plugin_macos.h @@ -4,6 +4,7 @@ #define NETDATA_PLUGIN_MACOS_H 1 #include "daemon/common.h" +#include "collectors/utils.h" int do_macos_sysctl(int update_every, usec_t dt); int do_macos_mach_smi(int update_every, usec_t dt); diff --git a/collectors/plugins.d/local_listeners.c b/collectors/plugins.d/local_listeners.c index a39de79748..159b25f8a4 100644 --- a/collectors/plugins.d/local_listeners.c +++ b/collectors/plugins.d/local_listeners.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #include "libnetdata/libnetdata.h" #include "libnetdata/required_dummies.h" diff --git a/collectors/plugins.d/plugins_d.c b/collectors/plugins.d/plugins_d.c index 6a235b4e62..1e9d71ee1a 100644 --- a/collectors/plugins.d/plugins_d.c +++ b/collectors/plugins.d/plugins_d.c @@ -288,11 +288,11 @@ void *pluginsd_main(void *ptr) strncpyz(cd->filename, file->d_name, FILENAME_MAX); snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", directory_name, cd->filename); - cd->host = localhost; + cd->host = rrdb.localhost; cd->unsafe.enabled = enabled; cd->unsafe.running = false; - cd->update_every = (int)config_get_number(cd->id, "update every", localhost->rrd_update_every); + cd->update_every = (int)config_get_number(cd->id, "update every", rrdb.localhost->update_every); cd->started_t = now_realtime_sec(); char *def = ""; diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index cda17710c5..e8189d67e2 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -204,7 +204,7 @@ static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, cons return NULL; } - RRDSET *st = rrdset_find(host, chart); + RRDSET *st = rrdset_find_by_id(host, chart); if (unlikely(!st)) netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.", rrdhost_hostname(host), chart, cmd); @@ -375,11 +375,38 @@ static inline PARSER_RC pluginsd_host_labels(char **words, size_t num_words, PAR PLUGINSD_KEYWORD_HOST_LABEL); } +static struct rrdhost_system_info *labels_to_system_info(DICTIONARY *labels) { + struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info)); + info->hops = 1; + + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type"); + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type"); + rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version"); + rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total"); + rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space"); + rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture"); + rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization"); + rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container"); + rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection"); + rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection"); + rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node"); + rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type"); + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch"); + rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist"); + + return info; +} + static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) { if(!parser->user.host_define.parsing_host) return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this"); - RRDHOST *host = rrdhost_find_or_create( + RRDHOST *host = rrdhost_get_or_create( string2str(parser->user.host_define.hostname), string2str(parser->user.host_define.hostname), parser->user.host_define.machine_guid_str, @@ -390,9 +417,9 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si NULL, program_name, program_version, - default_rrd_update_every, - default_rrd_history_entries, - default_rrd_memory_mode, + rrdb.default_update_every, + rrdb.default_rrd_history_entries, + default_storage_engine_id, default_health_enabled, default_rrdpush_enabled, default_rrdpush_destination, @@ -401,7 +428,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si default_rrdpush_enable_replication, default_rrdpush_seconds_to_replicate, default_rrdpush_replication_step, - rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels), + labels_to_system_info(parser->user.host_define.rrdlabels), false ); @@ -431,7 +458,7 @@ static inline PARSER_RC pluginsd_host(char **words, size_t num_words, PARSER *pa char *guid = get_word(words, num_words, 1); if(!guid || !*guid || strcmp(guid, "localhost") == 0) { - parser->user.host = localhost; + parser->user.host = rrdb.localhost; return PARSER_RC_OK; } diff --git a/collectors/proc.plugin/ipc.c b/collectors/proc.plugin/ipc.c index b166deba68..dd0a8733a3 100644 --- a/collectors/proc.plugin/ipc.c +++ b/collectors/proc.plugin/ipc.c @@ -327,7 +327,7 @@ int do_ipc(int update_every, usec_t dt) { , PLUGIN_PROC_NAME , "ipc" , NETDATA_CHART_PRIO_SYSTEM_IPC_SEMAPHORES - , localhost->rrd_update_every + , rrdb.localhost->update_every , RRDSET_TYPE_AREA ); rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -345,15 +345,15 @@ int do_ipc(int update_every, usec_t dt) { , PLUGIN_PROC_NAME , "ipc" , NETDATA_CHART_PRIO_SYSTEM_IPC_SEM_ARRAYS - , localhost->rrd_update_every + , rrdb.localhost->update_every , RRDSET_TYPE_AREA ); rd_arrays = rrddim_add(st_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } // variables - semaphores_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_max"); - arrays_max = rrdvar_custom_host_variable_add_and_acquire(localhost, "ipc_semaphores_arrays_max"); + semaphores_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_max"); + arrays_max = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "ipc_semaphores_arrays_max"); } struct stat stbuf; @@ -373,8 +373,8 @@ int do_ipc(int update_every, usec_t dt) { collector_error("Unable to fetch semaphore limits."); } else { - if(semaphores_max) rrdvar_custom_host_variable_set(localhost, semaphores_max, limits.semmns); - if(arrays_max) rrdvar_custom_host_variable_set(localhost, arrays_max, limits.semmni); + if(semaphores_max) rrdvar_custom_host_variable_set(rrdb.localhost, semaphores_max, limits.semmns); + if(arrays_max) rrdvar_custom_host_variable_set(rrdb.localhost, arrays_max, limits.semmni); st_arrays->red = limits.semmni; st_semaphores->red = limits.semmns; diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c index 74e3129370..019733cbca 100644 --- a/collectors/proc.plugin/plugin_proc.c +++ b/collectors/proc.plugin/plugin_proc.c @@ -161,7 +161,7 @@ void *proc_main(void *ptr) worker_register_job_name(i, proc_modules[i].dim); } - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); @@ -185,7 +185,7 @@ void *proc_main(void *ptr) netdata_log_debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name); worker_is_busy(i); - pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt); + pm->enabled = !pm->func(rrdb.localhost->update_every, hb_dt); } } diff --git a/collectors/proc.plugin/proc_mdstat.c b/collectors/proc.plugin/proc_mdstat.c index c3d1793cbe..cfebf37abc 100644 --- a/collectors/proc.plugin/proc_mdstat.c +++ b/collectors/proc.plugin/proc_mdstat.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "plugin_proc.h" +#include "collectors/utils.h" #define PLUGIN_PROC_MODULE_MDSTAT_NAME "/proc/mdstat" diff --git a/collectors/proc.plugin/proc_net_dev.c b/collectors/proc.plugin/proc_net_dev.c index 16881d1709..00a957fda8 100644 --- a/collectors/proc.plugin/proc_net_dev.c +++ b/collectors/proc.plugin/proc_net_dev.c @@ -1518,7 +1518,7 @@ void *netdev_main(void *ptr) netdata_thread_cleanup_push(netdev_main_cleanup, ptr); - usec_t step = localhost->rrd_update_every * USEC_PER_SEC; + usec_t step = rrdb.localhost->update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); @@ -1530,7 +1530,7 @@ void *netdev_main(void *ptr) break; worker_is_busy(0); - if(do_proc_net_dev(localhost->rrd_update_every, hb_dt)) + if(do_proc_net_dev(rrdb.localhost->update_every, hb_dt)) break; } diff --git a/collectors/proc.plugin/proc_net_netstat.c b/collectors/proc.plugin/proc_net_netstat.c index ce3068c0e5..250f8b3777 100644 --- a/collectors/proc.plugin/proc_net_netstat.c +++ b/collectors/proc.plugin/proc_net_netstat.c @@ -1647,7 +1647,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) { arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors); arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti); - tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_max_connections"); + tcp_max_connections_var = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_max_connections"); } size_t lines, l, words; @@ -2741,7 +2741,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) { // snmp Tcp charts // this is smart enough to update it, only when it is changed - rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn); + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn); // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO && diff --git a/collectors/proc.plugin/proc_net_sockstat.c b/collectors/proc.plugin/proc_net_sockstat.c index e94b891ca8..d18ae1e547 100644 --- a/collectors/proc.plugin/proc_net_sockstat.c +++ b/collectors/proc.plugin/proc_net_sockstat.c @@ -32,9 +32,9 @@ static int read_tcp_mem(void) { *tcp_mem_high_threshold = NULL; if(unlikely(!tcp_mem_low_threshold)) { - tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_low"); - tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_pressure"); - tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(localhost, "tcp_mem_high"); + tcp_mem_low_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_low"); + tcp_mem_pressure_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_pressure"); + tcp_mem_high_threshold = rrdvar_custom_host_variable_add_and_acquire(rrdb.localhost, "tcp_mem_high"); } if(unlikely(!filename)) { @@ -60,9 +60,9 @@ static int read_tcp_mem(void) { // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high); - rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0); - rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0); - rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0); + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024.0); + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024.0); + rrdvar_custom_host_variable_set(rrdb.localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024.0); return 0; } @@ -81,9 +81,9 @@ static kernel_uint_t read