Obvious memory reductions (#15204)

* remove rd->update_every

* reduce amount of memory for RRDDIM

* reorgnize rrddim->db entries

* optimize rrdset and statsd

* optimize dictionaries

* RW_SPINLOCK for dictionaries

* fix codeql warning

* rw_spinlock improvements

* remove obsolete assertion

* fix crash on health_alarm_log_process()

* use RW_SPINLOCK for AVL trees

* add RW_SPINLOCK read/write trylock

* pgc and mrg now use rw_spinlocks; cache line optimizations for mrg

* thread tag of dbegnine init

* append created datafile, lockless

* make DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE friendly for lockless use

* thread cancelability in spinlocks; optimize thread cancelability management

* introduce a JudyL to index datafiles and use it during queries to quickly find the relevant files

* use the last timestamp of each journal file for indexing

* when the previous cannot be found, start from the beginning

* add more stats to PDC to trace routing easier

* rename spinlock functions

* fix for spinlock renames

* revert statsd socket statistics to size_t

* turn fatal into internal_fatal()

* show candidates always

* show connected status and connection attempts

1 files changed, 33 insertions, 25 deletions

diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h
index 69e4123544..ddb74ba102 100644
--- a/database/engine/rrdengine.h
+++ b/database/engine/rrdengine.h
@@ -34,31 +34,6 @@ struct rrdeng_cmd;
 #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"
 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
 
-typedef struct page_details_control {
-    struct rrdengine_instance *ctx;
-    struct metric *metric;
-
-    struct completion prep_completion;
-    struct completion page_completion;   // sync between the query thread and the workers
-
-    Pvoid_t page_list_JudyL;        // the list of page details
-    unsigned completed_jobs;        // the number of jobs completed last time the query thread checked
-    bool workers_should_stop;       // true when the query thread left and the workers should stop
-    bool prep_done;
-
-    SPINLOCK refcount_spinlock;     // spinlock to protect refcount
-    int32_t refcount;               // the number of workers currently working on this request + 1 for the query thread
-    size_t executed_with_gaps;
-
-    time_t start_time_s;
-    time_t end_time_s;
-    STORAGE_PRIORITY priority;
-
-    time_t optimal_end_time_s;
-} PDC;
-
-PDC *pdc_get(void);
-
 typedef enum __attribute__ ((__packed__)) {
     // final status for all pages
     // if a page does not have one of these, it is considered unroutable
@@ -99,6 +74,34 @@ typedef enum __attribute__ ((__packed__)) {
 
 #define PDC_PAGE_QUERY_GLOBAL_SKIP_LIST (PDC_PAGE_FAILED | PDC_PAGE_SKIP | PDC_PAGE_INVALID | PDC_PAGE_RELEASED)
 
+typedef struct page_details_control {
+    struct rrdengine_instance *ctx;
+    struct metric *metric;
+
+    struct completion prep_completion;
+    struct completion page_completion;   // sync between the query thread and the workers
+
+    Pvoid_t page_list_JudyL;        // the list of page details
+    unsigned completed_jobs;        // the number of jobs completed last time the query thread checked
+    bool workers_should_stop;       // true when the query thread left and the workers should stop
+    bool prep_done;
+
+    PDC_PAGE_STATUS common_status;
+    size_t pages_to_load_from_disk;
+
+    SPINLOCK refcount_spinlock;     // spinlock to protect refcount
+    int32_t refcount;               // the number of workers currently working on this request + 1 for the query thread
+    size_t executed_with_gaps;
+
+    time_t start_time_s;
+    time_t end_time_s;
+    STORAGE_PRIORITY priority;
+
+    time_t optimal_end_time_s;
+} PDC;
+
+PDC *pdc_get(void);
+
 struct page_details {
     struct {
         struct rrdengine_datafile *ptr;
@@ -362,6 +365,11 @@ struct rrdengine_instance {
     } datafiles;
 
     struct {
+        RW_SPINLOCK spinlock;
+        Pvoid_t JudyL;
+    } njfv2idx;
+
+    struct {
         unsigned last_fileno;                       // newest index of datafile and journalfile
         unsigned last_flush_fileno;                 // newest index of datafile received data