From afe1b704857a7307547341a1027c019bbe68e910 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Thu, 13 Oct 2022 08:05:15 +0300 Subject: dbengine free from RRDSET and RRDDIM (#13772) * dbengine free from RRDSET and RRDDIM * fix for excess parameters to query ops * add comment about ML * update_every from int to uint32_t * rrddim_mem storage engine working * fixes for update_every_s * working dbengine * a lot of changes in dbengine regarding timestamps * better logging of not sequential points * rrdset_done() now gives aligned timestamps for higher tiers * dont change the end_time of descriptors, because they cant be loaded back * fixes for cmake * fixes for db mode ram * Global counters for dbengine loading errors. Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data. * update lgtm config * fixes for 32-bit systems * update unittests * Don't try to find and create a host on the fly if not already in memory * Remove unused functions * print backtrace in case of fatal * always set ctx to page_index * detect ctx and metric uuid discrepancies * use legacy uuid if multihost is not available * fix for last commit * prevent repeating log * Do not try to access archived charts when executing a data query * Remove unused function * log inconsistent collections once every 10 mins Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> --- web/api/formatters/rrd2json.c | 2 +- web/api/queries/query.c | 17 ++++++++++++++--- web/api/tests/valid_urls.c | 13 ------------- web/api/tests/web_api.c | 12 ------------ web/api/web_api_v1.c | 6 ------ web/server/web_client.c | 8 -------- 6 files changed, 15 insertions(+), 43 deletions(-) (limited to 'web') diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 09655d281a..abc350a1aa 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -27,7 +27,7 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc if(archive_mode) { STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode); if (eng) - eng->api.free(temp_rd->tiers[tier]->db_metric_handle); + eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle); } onewayalloc_freez(owa, temp_rd->tiers[tier]); diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 44cfa5ab87..79f10a52ef 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -966,7 +966,7 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti ops->tier = ops->plan.data[plan_id].tier; ops->tier_ptr = ops->rd->tiers[ops->tier]; - ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch); + ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before); ops->next_metric = ops->tier_ptr->query_ops.next_metric; ops->is_finished = ops->tier_ptr->query_ops.is_finished; ops->finalize = ops->tier_ptr->query_ops.finalize; @@ -1156,6 +1156,10 @@ static inline void rrd2rrdr_do_dimension( , time_t after_wanted , time_t before_wanted ){ +// bool debug_this = false; +// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0) +// debug_this = true; + time_t max_date = 0, min_date = 0; @@ -1210,6 +1214,9 @@ static inline void rrd2rrdr_do_dimension( new_point = QUERY_POINT_EMPTY; new_point.start_time = last1_point.end_time; new_point.end_time = now_end_time; +// +// if(debug_this) info("QUERY: is finished() returned true"); +// break; } @@ -1225,6 +1232,10 @@ static inline void rrd2rrdr_do_dimension( new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0; query_point_set_id(new_point, ops.db_total_points_read); +// if(debug_this) +// info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld", +// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted); +// // set the right value to the point we got if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) { @@ -1268,7 +1279,7 @@ static inline void rrd2rrdr_do_dimension( // check if the db is advancing the query if(unlikely(new_point.end_time <= last1_point.end_time)) { - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld", + internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld", rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time, last1_point.id, last1_point.end_time, now_start_time, now_end_time); @@ -1466,7 +1477,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) { long before_wanted = smaller_tier_last_time; struct rrddim_tier *tmp = rd->tiers[tr]; - tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE); + tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted); size_t points = 0; diff --git a/web/api/tests/valid_urls.c b/web/api/tests/valid_urls.c index 91cd19b096..8a2a87f105 100644 --- a/web/api/tests/valid_urls.c +++ b/web/api/tests/valid_urls.c @@ -19,19 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host) return NULL; } - -RRDHOST *sql_create_host_by_uuid(char *hostname) -{ - (void) hostname; - return NULL; -} - -RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname) -{ - (void) hostname; - return NULL; -} - void repr(char *result, int result_size, char const *buf, int size) { int n; diff --git a/web/api/tests/web_api.c b/web/api/tests/web_api.c index fd9a86ef6d..93e6454ee8 100644 --- a/web/api/tests/web_api.c +++ b/web/api/tests/web_api.c @@ -19,18 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host) return NULL; } -RRDHOST *sql_create_host_by_uuid(char *hostname) -{ - (void) hostname; - return NULL; -} - -RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname) -{ - (void) hostname; - return NULL; -} - void repr(char *result, int result_size, char const *buf, int size) { int n; diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index b58bb9b88b..87ff201f44 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -721,10 +721,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c if (likely(context_param_list && context_param_list->rd)) // Just set the first one st = context_param_list->rd->rrdset; - else { - if (!chart_label_key && !chart_labels_filter) - sql_build_context_param_list(owa, &context_param_list, host, context, NULL); - } } else { st = rrdset_find(host, chart); @@ -732,8 +728,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c st = rrdset_find_byname(host, chart); if (likely(st)) st->last_accessed_time = now_realtime_sec(); - else - sql_build_context_param_list(owa, &context_param_list, host, NULL, chart); } if (!st) { diff --git a/web/server/web_client.c b/web/server/web_client.c index ff485384f7..ff87041836 100644 --- a/web/server/web_client.c +++ b/web/server/web_client.c @@ -1308,14 +1308,6 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch host = rrdhost_find_by_hostname(tok); if (!host) host = rrdhost_find_by_guid(tok); - if (!host) { - host = sql_create_host_by_uuid(tok); - if (likely(host)) { - int rc = web_client_process_url(host, w, url); - free_temporary_host(host); - return rc; - } - } if (host) return web_client_process_url(host, w, url); } -- cgit v1.2.3