Deduplicate all netdata strings (#13570)

* rrdfamily

* rrddim

* rrdset plugin and module names

* rrdset units

* rrdset type

* rrdset family

* rrdset title

* rrdset title more

* rrdset context

* rrdcalctemplate context and removal of context hash from rrdset

* strings statistics

* rrdset name

* rearranged members of rrdset

* eliminate rrdset name hash; rrdcalc chart converted to STRING

* rrdset id, eliminated rrdset hash

* rrdcalc, alarm_entry, alert_config and some of rrdcalctemplate

* rrdcalctemplate

* rrdvar

* eval_variable

* rrddimvar and rrdsetvar

* rrdhost hostname, os and tags

* fix master commits

* added thread cache; implemented string_dup without locks

* faster thread cache

* rrdset and rrddim now use dictionaries for indexing

* rrdhost now uses dictionary

* rrdfamily now uses DICTIONARY

* rrdvar using dictionary instead of AVL

* allocate the right size to rrdvar flag members

* rrdhost remaining char * members to STRING *

* better error handling on indexing

* strings now use a read/write lock to allow parallel searches to the index

* removed AVL support from dictionaries; implemented STRING with native Judy calls

* string releases should be negative

* only 31 bits are allowed for enum flags

* proper locking on strings

* string threading unittest and fixes

* fix lgtm finding

* fixed naming

* stream chart/dimension definitions at the beginning of a streaming session

* thread stack variable is undefined on thread cancel

* rrdcontext garbage collect per host on startup

* worker control in garbage collection

* relaxed deletion of rrdmetrics

* type checking on dictfe

* netdata chart to monitor rrdcontext triggers

* Group chart label updates

* rrdcontext better handling of collected rrdsets

* rrdpush incremental transmition of definitions should use as much buffer as possible

* require 1MB per chart

* empty the sender buffer before enabling metrics streaming

* fill up to 50% of buffer

* reset signaling metrics sending

* use the shared variable for status

* use separate host flag for enabling streaming of metrics

* make sure the flag is clear

* add logging for streaming

* add logging for streaming on buffer overflow

* circular_buffer proper sizing

* removed obsolete logs

* do not execute worker jobs if not necessary

* better messages about compression disabling

* proper use of flags and updating rrdset last access time every time the obsoletion flag is flipped

* monitor stream sender used buffer ratio

* Update exporting unit tests

* no need to compare label value with strcmp

* streaming send workers now monitor bandwidth

* workers now use strings

* streaming receiver monitors incoming bandwidth

* parser shift of worker ids

* minor fixes

* Group chart label updates

* Populate context with dimensions that have data

* Fix chart id

* better shift of parser worker ids

* fix for streaming compression

* properly count received bytes

* ensure LZ4 compression ring buffer does not wrap prematurely

* do not stream empty charts; do not process empty instances in rrdcontext

* need_to_send_chart_definition() does not need an rrdset lock any more

* rrdcontext objects are collected, after data have been written to the db

* better logging of RRDCONTEXT transitions

* always set all variables needed by the worker utilization charts

* implemented double linked list for most objects; eliminated alarm indexes from rrdhost; and many more fixes

* lockless strings design - string_dup() and string_freez() are totally lockless when they dont need to touch Judy - only Judy is protected with a read/write lock

* STRING code re-organization for clarity

* thread_cache improvements; double numbers precision on worker threads

* STRING_ENTRY now shadown STRING, so no duplicate definition is required; string_length() renamed to string_strlen() to follow the paradigm of all other functions, STRING internal statistics are now only compiled with NETDATA_INTERNAL_CHECKS

* rrdhost index by hostname now cleans up; aclk queries of archieved hosts do not index hosts

* Add index to speed up database context searches

* Removed last_updated optimization (was also buggy after latest merge with master)

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>

1 files changed, 12 insertions, 14 deletions

diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c
index b51ea317d3..1312aa5712 100644
--- a/database/sqlite/sqlite_aclk.c
+++ b/database/sqlite/sqlite_aclk.c
@@ -237,15 +237,14 @@ struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id)
 void aclk_sync_exit_all()
 {
     rrd_rdlock();
-    RRDHOST *host = localhost;
-    while(host) {
+    RRDHOST *host;
+    rrdhost_foreach_read(host) {
         struct aclk_database_worker_config *wc = host->dbsync_worker;
         if (wc) {
             wc->is_shutting_down = 1;
             (void) aclk_database_deq_cmd(wc);
             uv_cond_signal(&wc->cmd_cond);
         }
-        host = host->next;
     }
     rrd_unlock();
 
@@ -320,7 +319,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
     char node_str[UUID_STR_LEN] = "<none>";
     if (likely(host->node_id))
         uuid_unparse_lower(*host->node_id, node_str);
-    internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", host->hostname, host->machine_guid, node_str);
+    internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str);
 #endif
     return 0;
 }
@@ -335,7 +334,7 @@ int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)
 
     uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str);
 
-    RRDHOST *host = rrdhost_find_by_guid(uuid_str, 0);
+    RRDHOST *host = rrdhost_find_by_guid(uuid_str);
     if (host == localhost)
         return 0;
 
@@ -526,7 +525,7 @@ void aclk_database_worker(void *arg)
 
     char threadname[NETDATA_THREAD_NAME_MAX+1];
     if (wc->host)
-        snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname);
+        snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host));
     else {
         snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->uuid_str);
         threadname[11] = '\0';
@@ -705,14 +704,14 @@ void aclk_database_worker(void *arg)
                 case ACLK_DATABASE_TIMER:
                     if (unlikely(localhost && !wc->host && !wc->is_orphan)) {
                         if (claimed()) {
-                            wc->host = rrdhost_find_by_guid(wc->host_guid, 0);
+                            wc->host = rrdhost_find_by_guid(wc->host_guid);
                             if (wc->host) {
-                                info("HOST %s (%s) detected as active", wc->host->hostname, wc->host_guid);
-                                snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", wc->host->hostname);
+                                info("HOST %s (%s) detected as active", rrdhost_hostname(wc->host), wc->host_guid);
+                                snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "AS_%s", rrdhost_hostname(wc->host));
                                 uv_thread_set_name_np(wc->thread, threadname);
                                 wc->host->dbsync_worker = wc;
                                 if (unlikely(!wc->hostname))
-                                    wc->hostname = strdupz(wc->host->hostname);
+                                    wc->hostname = strdupz(rrdhost_hostname(wc->host));
                                 aclk_del_worker_thread(wc);
                                 wc->node_info_send = 1;
                             }
@@ -844,7 +843,7 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
         uuid_unparse_lower(*node_id, wc->node_id);
     if (likely(host)) {
         host->dbsync_worker = (void *)wc;
-        wc->hostname = strdupz(host->hostname);
+        wc->hostname = strdupz(rrdhost_hostname(host));
     }
     else
         wc->hostname = get_hostname_by_node_id(wc->node_id);
@@ -1039,12 +1038,11 @@ void aclk_data_rotated(void)
 
     time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY;
     rrd_rdlock();
-    RRDHOST *this_host = localhost;
-    while (this_host) {
+    RRDHOST *this_host;
+    rrdhost_foreach_read(this_host) {
         struct aclk_database_worker_config *wc = this_host->dbsync_worker;
         if (wc)
             wc->rotation_after = next_rotation_time;
-        this_host = this_host->next;
     }
     rrd_unlock();