dbengine free from RRDSET and RRDDIM (#13772)

* dbengine free from RRDSET and RRDDIM

* fix for excess parameters to query ops

* add comment about ML

* update_every from int to uint32_t

* rrddim_mem storage engine working

* fixes for update_every_s

* working dbengine

* a lot of changes in dbengine regarding timestamps

* better logging of not sequential points

* rrdset_done() now gives aligned timestamps for higher tiers

* dont change the end_time of descriptors, because they cant be loaded back

* fixes for cmake

* fixes for db mode ram

* Global counters for dbengine loading errors.
Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data.

* update lgtm config

* fixes for 32-bit systems

* update unittests

* Don't try to find and create a host on the fly if not already in memory

* Remove unused functions

* print backtrace in case of fatal

* always set ctx to page_index

* detect ctx and metric uuid discrepancies

* use legacy uuid if multihost is not available

* fix for last commit

* prevent repeating log

* Do not try to access archived charts when executing a data query

* Remove unused function

* log inconsistent collections once every 10 mins

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 19 insertions, 9 deletions

diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h
index 9a2e005831..082367bbba 100644
--- a/database/engine/rrdengine.h
+++ b/database/engine/rrdengine.h
@@ -37,29 +37,25 @@ struct rrdengine_instance;
 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
 
 struct rrdeng_collect_handle {
-    struct rrdeng_metric_handle *metric_handle;
+    struct pg_cache_page_index *page_index;
     struct rrdeng_page_descr *descr;
     unsigned long page_correlation_id;
-    struct rrdengine_instance *ctx;
     // set to 1 when this dimension is not page aligned with the other dimensions in the chart
     uint8_t unaligned_page;
 };
 
 struct rrdeng_query_handle {
-    struct rrdeng_metric_handle *metric_handle;
     struct rrdeng_page_descr *descr;
     struct rrdengine_instance *ctx;
     struct pg_cache_page_index *page_index;
-    time_t next_page_time;
-    time_t now;
+    time_t wanted_start_time_s;
+    time_t now_s;
     unsigned position;
     unsigned entries;
-    TIER_QUERY_FETCH tier_query_fetch_type;
     storage_number *page;
-    usec_t page_end_time;
+    usec_t page_end_time_ut;
     uint32_t page_length;
-    usec_t dt;
-    time_t dt_sec;
+    time_t dt_s;
 };
 
 typedef enum {
@@ -230,6 +226,15 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele
 #define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
 #define QUIESCED    (2) /* is set after all threads have finished running */
 
+typedef enum {
+    LOAD_ERRORS_PAGE_FLIPPED_TIME = 0,
+    LOAD_ERRORS_PAGE_EQUAL_TIME = 1,
+    LOAD_ERRORS_PAGE_ZERO_ENTRIES = 2,
+    LOAD_ERRORS_PAGE_UPDATE_ZERO = 3,
+    LOAD_ERRORS_PAGE_FLEXY_TIME = 4,
+    LOAD_ERRORS_DROPPED_EXTENT = 5,
+} INVALID_PAGE_ID;
+
 struct rrdengine_instance {
     struct metalog_instance *metalog_ctx;
     struct rrdengine_worker_config worker_config;
@@ -254,6 +259,11 @@ struct rrdengine_instance {
     uint8_t page_type; /* Default page type for this context */
 
     struct rrdengine_statistics stats;
+
+    struct {
+        size_t counter;
+        usec_t latest_end_time_ut;
+    } load_errors[6];
 };
 
 void *dbengine_page_alloc(void);