dbengine free from RRDSET and RRDDIM (#13772)

* dbengine free from RRDSET and RRDDIM

* fix for excess parameters to query ops

* add comment about ML

* update_every from int to uint32_t

* rrddim_mem storage engine working

* fixes for update_every_s

* working dbengine

* a lot of changes in dbengine regarding timestamps

* better logging of not sequential points

* rrdset_done() now gives aligned timestamps for higher tiers

* dont change the end_time of descriptors, because they cant be loaded back

* fixes for cmake

* fixes for db mode ram

* Global counters for dbengine loading errors.
Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data.

* update lgtm config

* fixes for 32-bit systems

* update unittests

* Don't try to find and create a host on the fly if not already in memory

* Remove unused functions

* print backtrace in case of fatal

* always set ctx to page_index

* detect ctx and metric uuid discrepancies

* use legacy uuid if multihost is not available

* fix for last commit

* prevent repeating log

* Do not try to access archived charts when executing a data query

* Remove unused function

* log inconsistent collections once every 10 mins

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

6 files changed, 15 insertions, 43 deletions

diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 09655d281a..abc350a1aa 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -27,7 +27,7 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
         if(archive_mode) {
             STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
             if (eng)
-                eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
+                eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle);
         }
 
         onewayalloc_freez(owa, temp_rd->tiers[tier]);
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 44cfa5ab87..79f10a52ef 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -966,7 +966,7 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
 
     ops->tier = ops->plan.data[plan_id].tier;
     ops->tier_ptr = ops->rd->tiers[ops->tier];
-    ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
+    ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
     ops->next_metric = ops->tier_ptr->query_ops.next_metric;
     ops->is_finished = ops->tier_ptr->query_ops.is_finished;
     ops->finalize = ops->tier_ptr->query_ops.finalize;
@@ -1156,6 +1156,10 @@ static inline void rrd2rrdr_do_dimension(
     , time_t after_wanted
     , time_t before_wanted
 ){
+//    bool debug_this = false;
+//    if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
+//        debug_this = true;
+
     time_t max_date = 0,
            min_date = 0;
 
@@ -1210,6 +1214,9 @@ static inline void rrd2rrdr_do_dimension(
                 new_point = QUERY_POINT_EMPTY;
                 new_point.start_time = last1_point.end_time;
                 new_point.end_time   = now_end_time;
+//
+//                if(debug_this) info("QUERY: is finished() returned true");
+//
                 break;
             }
 
@@ -1225,6 +1232,10 @@ static inline void rrd2rrdr_do_dimension(
                 new_point.anomaly    = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
                 query_point_set_id(new_point, ops.db_total_points_read);
 
+//                if(debug_this)
+//                    info("QUERY: got point %zu, from time %ld to %ld   //   now from %ld to %ld   //   query from %ld to %ld",
+//                         new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
+//
                 // set the right value to the point we got
                 if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
 
@@ -1268,7 +1279,7 @@ static inline void rrd2rrdr_do_dimension(
 
             // check if the db is advancing the query
             if(unlikely(new_point.end_time <= last1_point.end_time)) {
-                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
+                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
                                rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time,
                                last1_point.id, last1_point.end_time, now_start_time, now_end_time);
 
@@ -1466,7 +1477,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
         long before_wanted = smaller_tier_last_time;
 
         struct rrddim_tier *tmp = rd->tiers[tr];
-        tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
+        tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
 
         size_t points = 0;
 
diff --git a/web/api/tests/valid_urls.c b/web/api/tests/valid_urls.c
index 91cd19b096..8a2a87f105 100644
--- a/web/api/tests/valid_urls.c
+++ b/web/api/tests/valid_urls.c
@@ -19,19 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;
diff --git a/web/api/tests/web_api.c b/web/api/tests/web_api.c
index fd9a86ef6d..93e6454ee8 100644
--- a/web/api/tests/web_api.c
+++ b/web/api/tests/web_api.c
@@ -19,18 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index b58bb9b88b..87ff201f44 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -721,10 +721,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
 
         if (likely(context_param_list && context_param_list->rd))  // Just set the first one
             st = context_param_list->rd->rrdset;
-        else {
-            if (!chart_label_key && !chart_labels_filter)
-                sql_build_context_param_list(owa, &context_param_list, host, context, NULL);
-        }
     }
     else {
         st = rrdset_find(host, chart);
@@ -732,8 +728,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
             st = rrdset_find_byname(host, chart);
         if (likely(st))
             st->last_accessed_time = now_realtime_sec();
-        else
-            sql_build_context_param_list(owa, &context_param_list, host, NULL, chart);
     }
 
     if (!st) {
diff --git a/web/server/web_client.c b/web/server/web_client.c
index ff485384f7..ff87041836 100644
--- a/web/server/web_client.c
+++ b/web/server/web_client.c
@@ -1308,14 +1308,6 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
         host = rrdhost_find_by_hostname(tok);
         if (!host)
             host = rrdhost_find_by_guid(tok);
-        if (!host) {
-            host = sql_create_host_by_uuid(tok);
-            if (likely(host)) {
-                int rc = web_client_process_url(host, w, url);
-                free_temporary_host(host);
-                return rc;
-            }
-        }
         if (host) return web_client_process_url(host, w, url);
     }