Obvious memory reductions (#15204)

* remove rd->update_every

* reduce amount of memory for RRDDIM

* reorgnize rrddim->db entries

* optimize rrdset and statsd

* optimize dictionaries

* RW_SPINLOCK for dictionaries

* fix codeql warning

* rw_spinlock improvements

* remove obsolete assertion

* fix crash on health_alarm_log_process()

* use RW_SPINLOCK for AVL trees

* add RW_SPINLOCK read/write trylock

* pgc and mrg now use rw_spinlocks; cache line optimizations for mrg

* thread tag of dbegnine init

* append created datafile, lockless

* make DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE friendly for lockless use

* thread cancelability in spinlocks; optimize thread cancelability management

* introduce a JudyL to index datafiles and use it during queries to quickly find the relevant files

* use the last timestamp of each journal file for indexing

* when the previous cannot be found, start from the beginning

* add more stats to PDC to trace routing easier

* rename spinlock functions

* fix for spinlock renames

* revert statsd socket statistics to size_t

* turn fatal into internal_fatal()

* show candidates always

* show connected status and connection attempts

3 files changed, 21 insertions, 20 deletions

diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 435e727ec9..2ae20ab71f 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -1718,7 +1718,7 @@ static void dbengine2_statistics_charts(void) {
     cache_efficiency_stats = rrdeng_get_cache_efficiency_stats();
 
     mrg_stats_old = mrg_stats;
-    mrg_stats = mrg_get_statistics(main_mrg);
+    mrg_get_statistics(main_mrg, &mrg_stats);
 
     struct rrdeng_buffer_sizes buffers = rrdeng_get_buffer_sizes();
     size_t buffers_total_size = buffers.handles + buffers.xt_buf + buffers.xt_io + buffers.pdc + buffers.descriptors +
diff --git a/daemon/main.c b/daemon/main.c
index 91b68a1a16..36bb158b1a 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -60,7 +60,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
     SERVICE_THREAD *sth = NULL;
     pid_t tid = gettid();
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
     Pvoid_t *PValue = JudyLIns(&service_globals.pid_judy, tid, PJE0);
     if(!*PValue) {
         sth = callocz(1, sizeof(SERVICE_THREAD));
@@ -87,7 +87,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
     else {
         sth = *PValue;
     }
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 
     return sth;
 }
@@ -95,13 +95,13 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
 void service_exits(void) {
     pid_t tid = gettid();
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
     Pvoid_t *PValue = JudyLGet(service_globals.pid_judy, tid, PJE0);
     if(PValue) {
         freez(*PValue);
         JudyLDel(&service_globals.pid_judy, tid, PJE0);
     }
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 }
 
 bool service_running(SERVICE_TYPE service) {
@@ -124,7 +124,7 @@ bool service_running(SERVICE_TYPE service) {
 void service_signal_exit(SERVICE_TYPE service) {
     __atomic_and_fetch(&service_globals.running, ~(service), __ATOMIC_RELAXED);
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
 
     Pvoid_t *PValue;
     Word_t tid = 0;
@@ -133,14 +133,14 @@ void service_signal_exit(SERVICE_TYPE service) {
         SERVICE_THREAD *sth = *PValue;
 
         if((sth->services & service) && sth->request_quit_callback) {
-            netdata_spinlock_unlock(&service_globals.lock);
+            spinlock_unlock(&service_globals.lock);
             sth->request_quit_callback(sth->data);
-            netdata_spinlock_lock(&service_globals.lock);
+            spinlock_lock(&service_globals.lock);
             continue;
         }
     }
 
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 }
 
 static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {
@@ -187,7 +187,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
     {
         buffer_flush(thread_list);
 
-        netdata_spinlock_lock(&service_globals.lock);
+        spinlock_lock(&service_globals.lock);
 
         Pvoid_t *PValue;
         Word_t tid = 0;
@@ -217,15 +217,15 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
                 running_services |= sth->services & service;
 
                 if(sth->force_quit_callback) {
-                    netdata_spinlock_unlock(&service_globals.lock);
+                    spinlock_unlock(&service_globals.lock);
                     sth->force_quit_callback(sth->data);
-                    netdata_spinlock_lock(&service_globals.lock);
+                    spinlock_lock(&service_globals.lock);
                     continue;
                 }
             }
         }
 
-        netdata_spinlock_unlock(&service_globals.lock);
+        spinlock_unlock(&service_globals.lock);
     }
 
     service_signal_exit(service);
@@ -244,7 +244,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
         running_services = 0;
         buffer_flush(thread_list);
 
-        netdata_spinlock_lock(&service_globals.lock);
+        spinlock_lock(&service_globals.lock);
 
         Pvoid_t *PValue;
         Word_t tid = 0;
@@ -262,7 +262,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
             }
         }
 
-        netdata_spinlock_unlock(&service_globals.lock);
+        spinlock_unlock(&service_globals.lock);
 
         if(running) {
             log_countdown_ut -= (log_countdown_ut >= sleep_ut) ? sleep_ut : log_countdown_ut;
diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index fa3fa847f2..f22d94b9b5 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -1334,13 +1334,14 @@ int run_test(struct test *test)
     int errors = 0;
 
     if(st->counter != test->result_entries) {
-        fprintf(stderr, "    %s stored %zu entries, but we were expecting %lu, ### E R R O R ###\n", test->name, st->counter, test->result_entries);
+        fprintf(stderr, "    %s stored %u entries, but we were expecting %lu, ### E R R O R ###\n",
+                test->name, st->counter, test->result_entries);
         errors++;
     }
 
     unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;
     for(c = 0 ; c < max ; c++) {
-        NETDATA_DOUBLE v = unpack_storage_number(rd->db[c]);
+        NETDATA_DOUBLE v = unpack_storage_number(rd->db.data[c]);
         NETDATA_DOUBLE n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
         int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
         fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
@@ -1352,7 +1353,7 @@ int run_test(struct test *test)
         if(!same) errors++;
 
         if(rd2) {
-            v = unpack_storage_number(rd2->db[c]);
+            v = unpack_storage_number(rd2->db.data[c]);
             n = test->results2[c];
             same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
             fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
@@ -1602,7 +1603,7 @@ int unit_test(long delay, long shift)
         fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
 
         rrddim_foreach_read(rd, st) {
-            sn = rd->db[c];
+            sn = rd->db.data[c];
             cn = unpack_storage_number(sn);
             fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ")   ->   ", rrddim_id(rd), cn, sn);
 
@@ -1821,7 +1822,7 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number
     rd->last_collected_time.tv_sec = now;
     rd->last_collected_time.tv_usec = 0;
     rd->collected_value = value;
-    rd->updated = 1;
+    rrddim_set_updated(rd);
 
     rd->collections_counter++;