diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-09-05 19:31:06 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-05 19:31:06 +0300 |
commit | 5e1b95cf92168c4df74586fb4430dc284806da82 (patch) | |
tree | f42077d8b02eaf316683453a7474bd1f599a833d | |
parent | 544aef1fde6e79ac57d2dea85d3f063076d7f885 (diff) |
Deduplicate all netdata strings (#13570)
* rrdfamily
* rrddim
* rrdset plugin and module names
* rrdset units
* rrdset type
* rrdset family
* rrdset title
* rrdset title more
* rrdset context
* rrdcalctemplate context and removal of context hash from rrdset
* strings statistics
* rrdset name
* rearranged members of rrdset
* eliminate rrdset name hash; rrdcalc chart converted to STRING
* rrdset id, eliminated rrdset hash
* rrdcalc, alarm_entry, alert_config and some of rrdcalctemplate
* rrdcalctemplate
* rrdvar
* eval_variable
* rrddimvar and rrdsetvar
* rrdhost hostname, os and tags
* fix master commits
* added thread cache; implemented string_dup without locks
* faster thread cache
* rrdset and rrddim now use dictionaries for indexing
* rrdhost now uses dictionary
* rrdfamily now uses DICTIONARY
* rrdvar using dictionary instead of AVL
* allocate the right size to rrdvar flag members
* rrdhost remaining char * members to STRING *
* better error handling on indexing
* strings now use a read/write lock to allow parallel searches to the index
* removed AVL support from dictionaries; implemented STRING with native Judy calls
* string releases should be negative
* only 31 bits are allowed for enum flags
* proper locking on strings
* string threading unittest and fixes
* fix lgtm finding
* fixed naming
* stream chart/dimension definitions at the beginning of a streaming session
* thread stack variable is undefined on thread cancel
* rrdcontext garbage collect per host on startup
* worker control in garbage collection
* relaxed deletion of rrdmetrics
* type checking on dictfe
* netdata chart to monitor rrdcontext triggers
* Group chart label updates
* rrdcontext better handling of collected rrdsets
* rrdpush incremental transmition of definitions should use as much buffer as possible
* require 1MB per chart
* empty the sender buffer before enabling metrics streaming
* fill up to 50% of buffer
* reset signaling metrics sending
* use the shared variable for status
* use separate host flag for enabling streaming of metrics
* make sure the flag is clear
* add logging for streaming
* add logging for streaming on buffer overflow
* circular_buffer proper sizing
* removed obsolete logs
* do not execute worker jobs if not necessary
* better messages about compression disabling
* proper use of flags and updating rrdset last access time every time the obsoletion flag is flipped
* monitor stream sender used buffer ratio
* Update exporting unit tests
* no need to compare label value with strcmp
* streaming send workers now monitor bandwidth
* workers now use strings
* streaming receiver monitors incoming bandwidth
* parser shift of worker ids
* minor fixes
* Group chart label updates
* Populate context with dimensions that have data
* Fix chart id
* better shift of parser worker ids
* fix for streaming compression
* properly count received bytes
* ensure LZ4 compression ring buffer does not wrap prematurely
* do not stream empty charts; do not process empty instances in rrdcontext
* need_to_send_chart_definition() does not need an rrdset lock any more
* rrdcontext objects are collected, after data have been written to the db
* better logging of RRDCONTEXT transitions
* always set all variables needed by the worker utilization charts
* implemented double linked list for most objects; eliminated alarm indexes from rrdhost; and many more fixes
* lockless strings design - string_dup() and string_freez() are totally lockless when they dont need to touch Judy - only Judy is protected with a read/write lock
* STRING code re-organization for clarity
* thread_cache improvements; double numbers precision on worker threads
* STRING_ENTRY now shadown STRING, so no duplicate definition is required; string_length() renamed to string_strlen() to follow the paradigm of all other functions, STRING internal statistics are now only compiled with NETDATA_INTERNAL_CHECKS
* rrdhost index by hostname now cleans up; aclk queries of archieved hosts do not index hosts
* Add index to speed up database context searches
* Removed last_updated optimization (was also buggy after latest merge with master)
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>
99 files changed, 4239 insertions, 3436 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 8143e5b694..82729885c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1498,6 +1498,7 @@ if(BUILD_TESTING) exporting/tests/netdata_doubles.c exporting/tests/system_doubles.c database/rrdlabels.c + database/rrdvar.c ) set(TEST_NAME exporting_engine) set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS) diff --git a/Makefile.am b/Makefile.am index 153731f978..e56a91f4fe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1252,6 +1252,7 @@ if ENABLE_UNITTESTS $(EXPORTING_ENGINE_FILES) \ $(LIBNETDATA_FILES) \ database/rrdlabels.c \ + database/rrdvar.c \ $(NULL) exporting_tests_exporting_engine_testdriver_CFLAGS = \ $(AM_CFLAGS) \ diff --git a/aclk/aclk.c b/aclk/aclk.c index cfd85ad3a3..4b8750064a 100644 --- a/aclk/aclk.c +++ b/aclk/aclk.c @@ -762,7 +762,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd) node_instance_creation_t node_instance_creation = { .claim_id = localhost->aclk_state.claimed_id, .hops = host->system_info->hops, - .hostname = host->hostname, + .hostname = rrdhost_hostname(host), .machine_guid = host->machine_guid }; create_query->data.bin_payload.payload = generate_node_instance_creation(&create_query->data.bin_payload.size, &node_instance_creation); @@ -829,7 +829,7 @@ void aclk_send_node_instances() char host_id[UUID_STR_LEN]; uuid_unparse_lower(list->host_id, host_id); - RRDHOST *host = rrdhost_find_by_guid(host_id, 0); + RRDHOST *host = rrdhost_find_by_guid(host_id); struct capability caps[] = { { .name = "proto", .version = 1, .enabled = 1 }, { .name = "ml", .version = ml_capable(localhost), .enabled = host ? ml_enabled(host) : 0 }, @@ -867,7 +867,7 @@ void aclk_send_node_instances() rrdhost_aclk_state_unlock(localhost); info("Queuing registration for host=%s, hops=%d",(char*)node_instance_creation.machine_guid, list->hops); - freez(node_instance_creation.machine_guid); + freez((void *)node_instance_creation.machine_guid); aclk_queue_query(create_query); } freez(list->hostname); @@ -992,7 +992,7 @@ char *aclk_state(void) RRDHOST *host; rrd_rdlock(); rrdhost_foreach_read(host) { - buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, host->hostname); + buffer_sprintf(wb, "\n\n> Node Instance for mGUID: \"%s\" hostname \"%s\"\n", host->machine_guid, rrdhost_hostname(host)); buffer_strcat(wb, "\tClaimed ID: "); rrdhost_aclk_state_lock(host); @@ -1179,7 +1179,7 @@ char *aclk_state_json(void) rrdhost_foreach_read(host) { json_object *nodeinstance = json_object_new_object(); - tmp = json_object_new_string(host->hostname); + tmp = json_object_new_string(rrdhost_hostname(host)); json_object_object_add(nodeinstance, "hostname", tmp); tmp = json_object_new_string(host->machine_guid); diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 981c01965a..132d5fe18f 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -38,9 +38,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id) int res; uuid_t node_id_bin, host_id_bin; - rrd_rdlock(); - RRDHOST *host = find_host_by_node_id((char *) node_id); - rrd_unlock(); + RRDHOST *host = find_host_by_node_id((char *)node_id); if (host) return host; @@ -54,7 +52,7 @@ static RRDHOST *node_id_2_rrdhost(const char *node_id) return NULL; } uuid_unparse_lower(host_id_bin, host_id); - return rrdhost_find_by_guid(host_id, 0); + return rrdhost_find_by_guid(host_id); } #define NODE_ID_QUERY "/node/" diff --git a/aclk/aclk_rx_msgs.c b/aclk/aclk_rx_msgs.c index e6ed332cc5..b42f010f9e 100644 --- a/aclk/aclk_rx_msgs.c +++ b/aclk/aclk_rx_msgs.c @@ -274,7 +274,7 @@ int create_node_instance_result(const char *msg, size_t msg_len) .node_id = res.node_id }; - RRDHOST *host = rrdhost_find_by_guid(res.machine_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(res.machine_guid); if (host) { // not all host must have RRDHOST struct created for them // if they never connected during runtime of agent @@ -527,7 +527,7 @@ unsigned int aclk_init_rx_msg_handlers(void) return i; } -void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic) +void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t msg_len, const char *topic __maybe_unused) { if (aclk_stats_enabled) { ACLK_STATS_LOCK; diff --git a/aclk/schema-wrappers/node_creation.h b/aclk/schema-wrappers/node_creation.h index 190ccb4d6e..7a8c7f7c76 100644 --- a/aclk/schema-wrappers/node_creation.h +++ b/aclk/schema-wrappers/node_creation.h @@ -8,9 +8,9 @@ extern "C" { #endif typedef struct { - char* claim_id; - char* machine_guid; - char* hostname; + const char *claim_id; + const char *machine_guid; + const char *hostname; int32_t hops; } node_instance_creation_t; diff --git a/aclk/schema-wrappers/node_info.h b/aclk/schema-wrappers/node_info.h index e8ac2d7c60..de4ade78aa 100644 --- a/aclk/schema-wrappers/node_info.h +++ b/aclk/schema-wrappers/node_info.h @@ -19,41 +19,27 @@ struct machine_learning_info { }; struct aclk_node_info { - char *name; - |