From cb7af25c09d8775d1967cb0553268075cda868d4 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 19 Sep 2022 23:46:13 +0300 Subject: RRD structures managed by dictionaries (#13646) * rrdset - in progress * rrdset optimal constructor; rrdset conflict * rrdset final touches * re-organization of rrdset object members * prevent use-after-free * dictionary dfe supports also counting of iterations * rrddim managed by dictionary * rrd.h cleanup * DICTIONARY_ITEM now is referencing actual dictionary items in the code * removed rrdset linked list * Revert "removed rrdset linked list" This reverts commit 690d6a588b4b99619c2c5e10f84e8f868ae6def5. * removed rrdset linked list * added comments * Switch chart uuid to static allocation in rrdset Remove unused functions * rrdset_archive() and friends... * always create rrdfamily * enable ml_free_dimension * rrddim_foreach done with dfe * most custom rrddim loops replaced with rrddim_foreach * removed accesses to rrddim->dimensions * removed locks that are no longer needed * rrdsetvar is now managed by the dictionary * set rrdset is rrdsetvar, fixes https://github.com/netdata/netdata/pull/13646#issuecomment-1242574853 * conflict callback of rrdsetvar now properly checks if it has to reset the variable * dictionary registered callbacks accept as first parameter the DICTIONARY_ITEM * dictionary dfe now uses internal counter to report; avoided excess variables defined with dfe * dictionary walkthrough callbacks get dictionary acquired items * dictionary reference counters that can be dupped from zero * added advanced functions for get and del * rrdvar managed by dictionaries * thread safety for rrdsetvar * faster rrdvar initialization * rrdvar string lengths should match in all add, del, get functions * rrdvar internals hidden from the rest of the world * rrdvar is now acquired throughout netdata * hide the internal structures of rrdsetvar * rrdsetvar is now acquired through out netdata * rrddimvar managed by dictionary; rrddimvar linked list removed; rrddimvar structures hidden from the rest of netdata * better error handling * dont create variables if not initialized for health * dont create variables if not initialized for health again * rrdfamily is now managed by dictionaries; references of it are acquired dictionary items * type checking on acquired objects * rrdcalc renaming of functions * type checking for rrdfamily_acquired * rrdcalc managed by dictionaries * rrdcalc double free fix * host rrdvars is always needed * attempt to fix deadlock 1 * attempt to fix deadlock 2 * Remove unused variable * attempt to fix deadlock 3 * snprintfz * rrdcalc index in rrdset fix * Stop storing active charts and computing chart hashes * Remove store active chart function * Remove compute chart hash function * Remove sql_store_chart_hash function * Remove store_active_dimension function * dictionary delayed destruction * formatting and cleanup * zero dictionary base on rrdsetvar * added internal error to log delayed destructions of dictionaries * typo in rrddimvar * added debugging info to dictionary * debug info * fix for rrdcalc keys being empty * remove forgotten unlock * remove deadlock * Switch to metadata version 5 and drop chart_hash chart_hash_map chart_active dimension_active v_chart_hash * SQL cosmetic changes * do not busy wait while destroying a referenced dictionary * remove deadlock * code cleanup; re-organization; * fast cleanup and flushing of dictionaries * number formatting fixes * do not delete configured alerts when archiving a chart * rrddim obsolete linked list management outside dictionaries * removed duplicate contexts call * fix crash when rrdfamily is not initialized * dont keep rrddimvar referenced * properly cleanup rrdvar * removed some locks * Do not attempt to cleanup chart_hash / chart_hash_map * rrdcalctemplate managed by dictionary * register callbacks on the right dictionary * removed some more locks * rrdcalc secondary index replaced with linked-list; rrdcalc labels updates are now executed by health thread * when looking up for an alarm look using both chart id and chart name * host initialization a bit more modular * init rrdlabels on host update * preparation for dictionary views * improved comment * unused variables without internal checks * service threads isolation and worker info * more worker info in service thread * thread cancelability debugging with internal checks * strings data races addressed; fixes https://github.com/netdata/netdata/issues/13647 * dictionary modularization * Remove unused SQL statement definition * unit-tested thread safety of dictionaries; removed data race conditions on dictionaries and strings; dictionaries now can detect if the caller is holds a write lock and automatically all the calls become their unsafe versions; all direct calls to unsafe version is eliminated * remove worker_is_idle() from the exit of service functions, because we lose the lock time between loops * rewritten dictionary to have 2 separate locks, one for indexing and another for traversal * Update collectors/cgroups.plugin/sys_fs_cgroup.c Co-authored-by: Vladimir Kobal * Update collectors/cgroups.plugin/sys_fs_cgroup.c Co-authored-by: Vladimir Kobal * Update collectors/proc.plugin/proc_net_dev.c Co-authored-by: Vladimir Kobal * fix memory leak in rrdset cache_dir * minor dictionary changes * dont use index locks in single threaded * obsolete dict option * rrddim options and flags separation; rrdset_done() optimization to keep array of reference pointers to rrddim; * fix jump on uninitialized value in dictionary; remove double free of cache_dir * addressed codacy findings * removed debugging code * use the private refcount on dictionaries * make dictionary item desctructors work on dictionary destruction; strictier control on dictionary API; proper cleanup sequence on rrddim; * more dictionary statistics * global statistics about dictionary operations, memory, items, callbacks * dictionary support for views - missing the public API * removed warning about unused parameter * chart and context name for cloud * chart and context name for cloud, again * dictionary statistics fixed; first implementation of dictionary views - not currently used * only the master can globally delete an item * context needs netdata prefix * fix context and chart it of spins * fix for host variables when health is not enabled * run garbage collector on item insert too * Fix info message; remove extra "using" * update dict unittest for new placement of garbage collector * we need RRDHOST->rrdvars for maintaining custom host variables * Health initialization needs the host->host_uuid * split STRING to its own files; no code changes other than that * initialize health unconditionally * unit tests do not pollute the global scope with their variables * Skip initialization when creating archived hosts on startup. When a child connects it will initialize properly Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> Co-authored-by: Vladimir Kobal --- web/api/badges/web_buffer_svg.c | 14 +++++++--- web/api/exporters/shell/allmetrics_shell.c | 21 ++++++--------- web/api/formatters/charts2json.c | 15 +++++------ web/api/formatters/json_wrapper.c | 25 +++++++----------- web/api/formatters/rrd2json.c | 11 +++----- web/api/formatters/rrdset2json.c | 17 ++++++------ web/api/queries/query.c | 40 ++++++++++++++-------------- web/api/queries/rrdr.c | 4 +-- web/api/queries/rrdr.h | 1 + web/api/queries/weights.c | 42 ++++++++++++------------------ web/api/web_api_v1.c | 8 +++--- 11 files changed, 89 insertions(+), 109 deletions(-) (limited to 'web') diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c index c4d9ced5f0..69f3d4367d 100644 --- a/web/api/badges/web_buffer_svg.c +++ b/web/api/badges/web_buffer_svg.c @@ -893,6 +893,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u int group = RRDR_GROUPING_AVERAGE; uint32_t options = 0x00000000; + const RRDCALC_ACQUIRED *rca = NULL; + RRDCALC *rc = NULL; + RRDSET *st = NULL; + while(url) { char *value = mystrsep(&url, "&"); if(!value || !*value) continue; @@ -957,7 +961,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u int scale = (scale_str && *scale_str)?str2i(scale_str):100; - RRDSET *st = rrdset_find(host, chart); + st = rrdset_find(host, chart); if(!st) st = rrdset_find_byname(host, chart); if(!st) { buffer_no_cacheable(w->response.data); @@ -967,9 +971,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u } st->last_accessed_time = now_realtime_sec(); - RRDCALC *rc = NULL; if(alarm) { - rc = rrdcalc_find(st, alarm); + rca = rrdcalc_from_rrdset_get(st, alarm); + rc = rrdcalc_acquired_to_rrdcalc(rca); + if (!rc) { buffer_no_cacheable(w->response.data); buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1, scale, 0, -1, -1, NULL, NULL); @@ -1143,7 +1148,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u ); } - cleanup: +cleanup: + rrdcalc_from_rrdset_release(st, rca); buffer_free(dimensions); return ret; } diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c index 184f44b188..0ffbac67b9 100644 --- a/web/api/exporters/shell/allmetrics_shell.c +++ b/web/api/exporters/shell/allmetrics_shell.c @@ -25,7 +25,6 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) { void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_shell(); SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); // for each chart RRDSET *st; @@ -39,8 +38,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", rrdset_id(st), rrdset_name(st)); if(rrdset_is_available_for_viewers(st)) { - rrdset_rdlock(st); - // for each dimension RRDDIM *rd; rrddim_foreach_read(rd, st) { @@ -55,22 +52,23 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ else { if(rd->multiplier < 0 || rd->divisor < 0) n = -n; n = roundndd(n); - if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n; + if(!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) total += n; buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, rrdset_units(st)); } } } + rrddim_foreach_done(rd); total = roundndd(total); buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, rrdset_units(st)); - rrdset_unlock(st); } } + rrdset_foreach_done(st); buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n"); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(!rc->rrdset) continue; char chart[SHELL_ELEMENT_MAX + 1]; @@ -90,8 +88,8 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status)); } + foreach_rrdcalc_in_rrdhost_done(rc); - rrdhost_unlock(host); simple_pattern_free(filter); } @@ -100,7 +98,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_json(); SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); buffer_strcat(wb, "{"); @@ -114,8 +111,6 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s continue; if(rrdset_is_available_for_viewers(st)) { - rrdset_rdlock(st); - buffer_sprintf( wb, "%s\n" @@ -132,7 +127,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s rrdset_family(st), rrdset_context(st), rrdset_units(st), - (int64_t)rrdset_last_entry_t_nolock(st)); + (int64_t)rrdset_last_entry_t(st)); chart_counter++; dimension_counter = 0; @@ -161,14 +156,14 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s dimension_counter++; } } + rrddim_foreach_done(rd); buffer_strcat(wb, "\n\t\t}\n\t}"); - rrdset_unlock(st); } } + rrdset_foreach_done(st); buffer_strcat(wb, "\n}"); - rrdhost_unlock(host); simple_pattern_free(filter); } diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 73e4247f4e..1fc20b4935 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -69,7 +69,6 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived ); c = 0; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) { if(c) buffer_strcat(wb, ","); @@ -82,13 +81,14 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived st->last_accessed_time = now; } } + rrdset_foreach_done(st); RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(rc->rrdset) alarms++; } - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); buffer_sprintf(wb , "\n\t}" @@ -150,9 +150,7 @@ struct array_printer { BUFFER *wb; }; -static int print_collector_callback(const char *name, void *entry, void *data) { - (void)name; - +static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct array_printer *ap = (struct array_printer *)data; BUFFER *wb = ap->wb; struct collector *col=(struct collector *) entry; @@ -167,12 +165,11 @@ static int print_collector_callback(const char *name, void *entry, void *data) { } void chartcollectors2json(RRDHOST *host, BUFFER *wb) { - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); RRDSET *st; char name[500]; time_t now = now_realtime_sec(); - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector col = { @@ -184,7 +181,7 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) { st->last_accessed_time = now; } } - rrdhost_unlock(host); + rrdset_foreach_done(st); struct array_printer ap = { .c = 0, .wb = wb diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 41aee34fb8..3ebe42c99f 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -7,9 +7,7 @@ struct value_output { BUFFER *wb; }; -static int value_list_output(const char *name, void *entry, void *data) { - (void)name; - +static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct value_output *ap = (struct value_output *)data; BUFFER *wb = ap->wb; char *output = (char *) entry; @@ -64,8 +62,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS sq[0] = '"'; } - if (should_lock) - rrdset_rdlock(r->st); buffer_sprintf(wb, "{\n" " %sapi%s: 1,\n" " %sid%s: %s%s%s,\n" @@ -83,8 +79,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS , kq, kq, sq, context_mode && temp_rd?rrdset_context(r->st):rrdset_name(r->st), sq , kq, kq, r->update_every , kq, kq, r->st->update_every - , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st)) - , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st)) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st)) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st)) , kq, kq, (uint32_t)r->before , kq, kq, (uint32_t)r->after , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq @@ -94,9 +90,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); - if (should_lock) - rrdset_unlock(r->st); - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -147,37 +140,37 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS struct value_output co = {.c = 0, .wb = wb}; - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrddim_id(rd), rrddim_name(rd)); int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrddim_id(rd), rrddim_name(rd)); dictionary_set(dict, name, output, len+1); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rrdset_id(rd->rrdset), rrdset_name(rd->rrdset)); dictionary_set(dict, name, output, len + 1); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); RRDSET *st; co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { st = rd->rrdset; if (st->rrdlabels) rrdlabels_walkthrough_read(st->rrdlabels, fill_formatted_callback, dict); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); buffer_strcat(wb, "],\n"); } diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 89efa05441..09655d281a 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -104,13 +104,11 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis (*param_list)->rd = NULL; } - RRDDIM *rd1; st->last_accessed_time = now_realtime_sec(); - rrdset_rdlock(st); - - (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st)); - (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); + (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st)); + (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st)); + RRDDIM *rd1; rrddim_foreach_read(rd1, st) { RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM)); rd->id = string_dup(rd1->id); @@ -124,8 +122,7 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis rd->next = (*param_list)->rd; (*param_list)->rd = rd; } - - rrdset_unlock(st); + rrddim_foreach_done(rd1); } void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index d4225c98c4..7758601e29 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -25,10 +25,8 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) // generate JSON for the /api/v1/chart API call void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile) { - rrdset_rdlock(st); - - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); buffer_sprintf( wb, @@ -47,7 +45,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"chart_type\": \"%s\",\n", rrdset_id(st), rrdset_name(st), - rrdset_type(st), + rrdset_parts_type(st), rrdset_family(st), rrdset_context(st), rrdset_title(st), @@ -90,7 +88,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor size_t dimensions = 0; RRDDIM *rd; rrddim_foreach_read(rd, st) { - if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; + if(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; memory += sizeof(RRDDIM) + rd->memsize; @@ -105,6 +103,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor dimensions++; } + rrddim_foreach_done(rd); if(dimensions_count) *dimensions_count += dimensions; if(memory_used) *memory_used += memory; @@ -121,7 +120,8 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); size_t alarms = 0; RRDCALC *rc; - foreach_rrdcalc_in_rrdset(st, rc) { + netdata_rwlock_rdlock(&st->alerts.rwlock); + DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) { buffer_sprintf( wb, "%s" @@ -136,6 +136,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor alarms++; } + netdata_rwlock_unlock(&st->alerts.rwlock); buffer_sprintf(wb, "\n\t\t\t}" ); @@ -148,6 +149,4 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_sprintf(wb, "\n\t\t}" ); - - rrdset_unlock(st); } diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 300a4429ce..44cfa5ab87 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -837,36 +837,38 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) { int ret = st->update_every; - if(unlikely(!st->dimensions)) + if(unlikely(!rrdset_number_of_dimensions(st))) return ret; - rrdset_rdlock(st); - int best_tier; + RRDDIM *first_rd = NULL; + rrddim_foreach_read(first_rd, st) break; rrddim_foreach_done(first_rd); + if(!first_rd) + return ret; + int best_tier; if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers) best_tier = tier; - else - best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted); + else { + best_tier = rrddim_find_best_tier_for_timeframe(first_rd, after_wanted, before_wanted, points_wanted); + } - if(!st->dimensions->tiers[best_tier]) { + if(!first_rd->tiers[best_tier]) { internal_error( true, "QUERY: tier %d on chart '%s', is not initialized", best_tier, rrdset_name(st)); } else { - ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every; + ret = (int)first_rd->tiers[best_tier]->tier_grouping * (int)st->update_every; if(unlikely(!ret)) { internal_error( true, "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d", - rrdset_name(st), st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); + rrdset_name(st), first_rd->tiers[best_tier]->tier_grouping, st->update_every); ret = st->update_every; } } - rrdset_unlock(st); - return ret; } @@ -1510,7 +1512,10 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r //, size_t before_slot , const char *msg ) { - netdata_rwlock_rdlock(&r->st->rrdset_rwlock); + + time_t first_entry_t = rrdset_first_entry_t(r->st); + time_t last_entry_t = rrdset_last_entry_t(r->st); + info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), " "after (got: %zu, want: %zu, req: %ld, db: %zu), " "before (got: %zu, want: %zu, req: %ld, db: %zu), " @@ -1532,19 +1537,19 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r , (size_t)r->after , (size_t)after_wanted , after_requested - , (size_t)rrdset_first_entry_t_nolock(r->st) + , (size_t)first_entry_t // before , (size_t)r->before , (size_t)before_wanted , before_requested - , (size_t)rrdset_last_entry_t_nolock(r->st) + , (size_t)last_entry_t // duration , (size_t)(r->before - r->after + r->st->update_every) , (size_t)(before_wanted - after_wanted + r->st->update_every) , before_requested - after_requested - , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every) + , (size_t)((last_entry_t - first_entry_t) + r->st->update_every) // slot /* @@ -1562,7 +1567,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r // message , msg ); - netdata_rwlock_unlock(&r->st->rrdset_rwlock); } #endif // NETDATA_INTERNAL_CHECKS @@ -1727,10 +1731,8 @@ RRDR *rrd2rrdr( if(!context_param_list) { relative_period_requested = true; - rrdset_rdlock(st); - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); - rrdset_unlock(st); + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); if(first_entry_t == 0 || last_entry_t == 0) { internal_error(true, "QUERY: chart without data detected on '%s'", rrdset_name(st)); diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index ecf4ca2ac3..09f1a12da3 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -111,7 +111,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa t = t->next; } } else - rrddim_foreach_read(rd, st) dimensions++; + dimensions = rrdset_number_of_dimensions(st); // create the rrdr RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n); @@ -121,7 +121,7 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa // set the hidden flag on hidden dimensions int c; for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) { - if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN))) + if (unlikely(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))) r->od[c] = RRDR_DIMENSION_HIDDEN; else r->od[c] = RRDR_DIMENSION_DEFAULT; diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 1c80e103fa..ffc378193c 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -44,6 +44,7 @@ typedef enum rrdr_options { // internal ones - not to be exposed to the API RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate + RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION } RRDR_OPTIONS; typedef enum rrdr_value_flag { diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 5ff997a26b..144b072915 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -65,10 +65,7 @@ struct register_result { struct register_result *next; // used to link contexts together }; -static void register_result_insert_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; - +static void register_result_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct register_result *t = (struct register_result *)value; if(t->chart_id) t->chart_id = strdupz(t->chart_id); @@ -76,9 +73,7 @@ static void register_result_insert_callback(const char *name, void *value, void if(t->dim_name) t->dim_name = strdupz(t->dim_name); } -static void register_result_delete_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; +static void register_result_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct register_result *t = (struct register_result *)value; freez((void *)t->chart_id); @@ -87,7 +82,7 @@ static void register_result_delete_callback(const char *name, void *value, void } static DICTIONARY *register_result_init() { - DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED); dictionary_register_insert_callback(results, register_result_insert_callback, results); dictionary_register_delete_callback(results, register_result_delete_callback, results); return results; @@ -261,11 +256,8 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w points, method, group, options, shifts, examined_dimensions, duration, stats); DICTIONARY *context_results = dictionary_create( - DICTIONARY_FLAG_SINGLE_THREADED - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_NAME_LINK_DONT_CLONE | + DICT_OPTION_DONT_OVERWRITE_VALUE); struct register_result *t; dfe_start_read(results, t) { @@ -605,7 +597,9 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, // for each dimension RRDDIM *d; int i; - for(i = 0, d = base_rrdr->st->dimensions; d && i < base_rrdr->d; i++, d = d->next) { + rrddim_foreach_read(d, base_rrdr->st) { + if(unlikely((int)d_dfe.counter >= base_rrdr->d)) break; + i = (int)d_dfe.counter; // d_counter is provided by the dictionary // skip the not evaluated ones if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN)) @@ -650,6 +644,7 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); } } + rrddim_foreach_done(d); cleanup: rrdr_free(owa, high_rrdr); @@ -676,7 +671,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, usec_t started_usec = now_realtime_usec(); RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { + rrddim_foreach_read(d, st) { usec_t now_usec = now_realtime_usec(); if(now_usec - started_usec > timeout * USEC_PER_MS) return examined_dimensions; @@ -766,6 +761,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, register_result(results, st, d, pcent, flags, stats, register_zero); } + rrddim_foreach_done(d); return examined_dimensions; } @@ -787,7 +783,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, usec_t started_usec = now_realtime_usec(); RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { + rrddim_foreach_read(d, st) { usec_t now_usec = now_realtime_usec(); if(now_usec - started_usec > timeout * USEC_PER_MS) return examined_dimensions; @@ -814,6 +810,7 @@ static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average)) register_result(results, st, d, average, 0, stats, register_zero); } + rrddim_foreach_done(d); return examined_dimensions; } @@ -853,7 +850,7 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { struct register_result *t; // count the dimensions - size_t dimensions = dictionary_stats_entries(results); + size_t dimensions = dictionary_entries(results); if(!dimensions) return 0; if(stats->max_base_high_ratio == 0.0) @@ -911,7 +908,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS WEIGHTS_STATS stats = {}; DICTIONARY *results = register_result_init(); - DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);; + DICTIONARY *charts = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE);; char *error = NULL; int resp = HTTP_RESP_OK; @@ -1003,14 +1000,13 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS // dont lock here and wait for results // get the charts and run mc after RRDSET *st; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { if(!contexts || simple_pattern_matches(contexts, rrdset_context(st))) dictionary_set(charts, rrdset_name(st), NULL, 0); } } - rrdhost_unlock(host); + rrdset_foreach_done(st); size_t examined_dimensions = 0; void *ptr; @@ -1030,11 +1026,9 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS goto cleanup; } - st = rrdset_find_byname(host, ptr_name); + st = rrdset_find_byname(host, ptr_dfe.name); // ptr_dfe.name is provided by dictionary if(!st) continue; - rrdset_rdlock(st); - switch(method) { case WEIGHTS_METHOD_ANOMALY_RATE: options |= RRDR_OPTION_ANOMALY_BIT; @@ -1066,8 +1060,6 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS &stats, register_zero); break; } - - rrdset_unlock(st); } dfe_done(ptr); diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 8063ffc7bc..2b8e2d8e77 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -710,14 +710,13 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); STRING *context_string = string_strdupz(context); - rrdhost_rdlock(host); rrdset_foreach_read(st1, host) { if (st1->context == context_string && (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_label_key_pattern, ':')) && (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->rrdlabels, chart_labels_filter_pattern, ':'))) build_context_param_list(owa, &context_param_list, st1); } - rrdhost_unlock(host); + rrdset_foreach_done(st1); string_freez(context_string); if (likely(context_param_list && context_param_list->rd)) // Just set the first one @@ -1054,8 +1053,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client * static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb) { int alarm_normal = 0, alarm_warn = 0, alarm_crit = 0; RRDCALC *rc; - rrdhost_rdlock(host); - foreach_rrdcalc_in_rrdhost(host, rc) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) continue; @@ -1070,7 +1068,7 @@ static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST alarm_normal++; } } - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); buffer_sprintf(wb, "\t\t\"normal\": %d,\n", alarm_normal); buffer_sprintf(wb, "\t\t\"warning\": %d,\n", alarm_warn); buffer_sprintf(wb, "\t\t\"critical\": %d\n", alarm_crit); -- cgit v1.2.3