DBENGINE v2 - improvements part 8 (#14319)

* cache 100 pages for each size our tiers need

* smarter page caching

* account the caching structures

* dynamic max number of cached pages

* make variables const to ensure they are not changed

* make sure replication timestamps do not go to the future

* replication now sends chart and dimension states atomically; replication receivers ignores chart and dimension states when rbegin is also ignored

* make sure all pages are flushed on shutdown

* take into account empty points too

* when recalculating retention update first_time_s on metrics only when they are bigger

* Report the datafile number we use to recalculate retention

* Report the datafile number we use to recalculate retention

* rotate db at startup

* make query plans overlap

* Calculate properly first time s

* updated event labels

* negative page caching fix

* Atempt to create missing tables on query failure

* Atempt to create missing tables on query failure (part 2)

* negative page caching for all gaps, to eliminate jv2 scans

* Fix unittest

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 13 insertions, 7 deletions

diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index e5dcd79719..f2c4852677 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -1018,15 +1018,16 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
     ops->is_finished = qm->plan.array[plan_id].is_finished;
     ops->finalize = qm->plan.array[plan_id].finalize;
     ops->current_plan = plan_id;
-    ops->current_plan_expire_time = qm->plan.array[plan_id].before;
+
+    if(plan_id + 1 < qm->plan.used && qm->plan.array[plan_id + 1].after < qm->plan.array[plan_id].before)
+        ops->current_plan_expire_time = qm->plan.array[plan_id + 1].after;
+    else
+        ops->current_plan_expire_time = qm->plan.array[plan_id].before;
 }
 
 static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) {
     QUERY_METRIC *qm = ops->qm;
 
-    internal_fatal(now < ops->current_plan_expire_time && now < qm->plan.array[ops->current_plan].before,
-                   "QUERY: switching query plan too early!");
-
     size_t old_plan = ops->current_plan;
 
     time_t next_plan_before_time;
@@ -1179,6 +1180,11 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
     }
 #endif
 
+    for(size_t p = 0; p < qm->plan.used ;p++) {
+        size_t tier = qm->plan.array[p].tier;
+        qm->plan.array[p].before += qm->tiers[tier].db_update_every_s - 1;
+    }
+
     query_planer_initialize_plans(ops);
     query_planer_activate_plan(ops, 0, 0);
 
@@ -1355,7 +1361,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
             }
 
             // check if the db is giving us zero duration points
-            if(unlikely(new_point.start_time == new_point.end_time)) {
+            if(unlikely(db_points_read_since_plan_switch > 1 && new_point.start_time == new_point.end_time)) {
                 internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu start time %ld, end time %ld, that are both equal",
                                qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time);
 
@@ -1363,8 +1369,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_
             }
 
             // check if the db is advancing the query
-            if(unlikely(new_point.end_time <= last1_point.end_time)) {
-                internal_error(db_points_read_since_plan_switch > 1,
+            if(unlikely(db_points_read_since_plan_switch > 1 && new_point.end_time <= last1_point.end_time)) {
+                internal_error(true,
                                "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu from %ld to %ld, now is %ld to %ld",
                                qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time,
                                last1_point.id, last1_point.start_time, last1_point.end_time, now_start_time, now_end_time);