From cb7af25c09d8775d1967cb0553268075cda868d4 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 19 Sep 2022 23:46:13 +0300 Subject: RRD structures managed by dictionaries (#13646) * rrdset - in progress * rrdset optimal constructor; rrdset conflict * rrdset final touches * re-organization of rrdset object members * prevent use-after-free * dictionary dfe supports also counting of iterations * rrddim managed by dictionary * rrd.h cleanup * DICTIONARY_ITEM now is referencing actual dictionary items in the code * removed rrdset linked list * Revert "removed rrdset linked list" This reverts commit 690d6a588b4b99619c2c5e10f84e8f868ae6def5. * removed rrdset linked list * added comments * Switch chart uuid to static allocation in rrdset Remove unused functions * rrdset_archive() and friends... * always create rrdfamily * enable ml_free_dimension * rrddim_foreach done with dfe * most custom rrddim loops replaced with rrddim_foreach * removed accesses to rrddim->dimensions * removed locks that are no longer needed * rrdsetvar is now managed by the dictionary * set rrdset is rrdsetvar, fixes https://github.com/netdata/netdata/pull/13646#issuecomment-1242574853 * conflict callback of rrdsetvar now properly checks if it has to reset the variable * dictionary registered callbacks accept as first parameter the DICTIONARY_ITEM * dictionary dfe now uses internal counter to report; avoided excess variables defined with dfe * dictionary walkthrough callbacks get dictionary acquired items * dictionary reference counters that can be dupped from zero * added advanced functions for get and del * rrdvar managed by dictionaries * thread safety for rrdsetvar * faster rrdvar initialization * rrdvar string lengths should match in all add, del, get functions * rrdvar internals hidden from the rest of the world * rrdvar is now acquired throughout netdata * hide the internal structures of rrdsetvar * rrdsetvar is now acquired through out netdata * rrddimvar managed by dictionary; rrddimvar linked list removed; rrddimvar structures hidden from the rest of netdata * better error handling * dont create variables if not initialized for health * dont create variables if not initialized for health again * rrdfamily is now managed by dictionaries; references of it are acquired dictionary items * type checking on acquired objects * rrdcalc renaming of functions * type checking for rrdfamily_acquired * rrdcalc managed by dictionaries * rrdcalc double free fix * host rrdvars is always needed * attempt to fix deadlock 1 * attempt to fix deadlock 2 * Remove unused variable * attempt to fix deadlock 3 * snprintfz * rrdcalc index in rrdset fix * Stop storing active charts and computing chart hashes * Remove store active chart function * Remove compute chart hash function * Remove sql_store_chart_hash function * Remove store_active_dimension function * dictionary delayed destruction * formatting and cleanup * zero dictionary base on rrdsetvar * added internal error to log delayed destructions of dictionaries * typo in rrddimvar * added debugging info to dictionary * debug info * fix for rrdcalc keys being empty * remove forgotten unlock * remove deadlock * Switch to metadata version 5 and drop chart_hash chart_hash_map chart_active dimension_active v_chart_hash * SQL cosmetic changes * do not busy wait while destroying a referenced dictionary * remove deadlock * code cleanup; re-organization; * fast cleanup and flushing of dictionaries * number formatting fixes * do not delete configured alerts when archiving a chart * rrddim obsolete linked list management outside dictionaries * removed duplicate contexts call * fix crash when rrdfamily is not initialized * dont keep rrddimvar referenced * properly cleanup rrdvar * removed some locks * Do not attempt to cleanup chart_hash / chart_hash_map * rrdcalctemplate managed by dictionary * register callbacks on the right dictionary * removed some more locks * rrdcalc secondary index replaced with linked-list; rrdcalc labels updates are now executed by health thread * when looking up for an alarm look using both chart id and chart name * host initialization a bit more modular * init rrdlabels on host update * preparation for dictionary views * improved comment * unused variables without internal checks * service threads isolation and worker info * more worker info in service thread * thread cancelability debugging with internal checks * strings data races addressed; fixes https://github.com/netdata/netdata/issues/13647 * dictionary modularization * Remove unused SQL statement definition * unit-tested thread safety of dictionaries; removed data race conditions on dictionaries and strings; dictionaries now can detect if the caller is holds a write lock and automatically all the calls become their unsafe versions; all direct calls to unsafe version is eliminated * remove worker_is_idle() from the exit of service functions, because we lose the lock time between loops * rewritten dictionary to have 2 separate locks, one for indexing and another for traversal * Update collectors/cgroups.plugin/sys_fs_cgroup.c Co-authored-by: Vladimir Kobal * Update collectors/cgroups.plugin/sys_fs_cgroup.c Co-authored-by: Vladimir Kobal * Update collectors/proc.plugin/proc_net_dev.c Co-authored-by: Vladimir Kobal * fix memory leak in rrdset cache_dir * minor dictionary changes * dont use index locks in single threaded * obsolete dict option * rrddim options and flags separation; rrdset_done() optimization to keep array of reference pointers to rrddim; * fix jump on uninitialized value in dictionary; remove double free of cache_dir * addressed codacy findings * removed debugging code * use the private refcount on dictionaries * make dictionary item desctructors work on dictionary destruction; strictier control on dictionary API; proper cleanup sequence on rrddim; * more dictionary statistics * global statistics about dictionary operations, memory, items, callbacks * dictionary support for views - missing the public API * removed warning about unused parameter * chart and context name for cloud * chart and context name for cloud, again * dictionary statistics fixed; first implementation of dictionary views - not currently used * only the master can globally delete an item * context needs netdata prefix * fix context and chart it of spins * fix for host variables when health is not enabled * run garbage collector on item insert too * Fix info message; remove extra "using" * update dict unittest for new placement of garbage collector * we need RRDHOST->rrdvars for maintaining custom host variables * Health initialization needs the host->host_uuid * split STRING to its own files; no code changes other than that * initialize health unconditionally * unit tests do not pollute the global scope with their variables * Skip initialization when creating archived hosts on startup. When a child connects it will initialize properly Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> Co-authored-by: Vladimir Kobal --- database/engine/journalfile.c | 2 +- database/engine/rrdengineapi.c | 2 +- database/ram/rrddim_mem.c | 2 +- database/rrd.h | 449 +++++++------- database/rrdcalc.c | 869 ++++++++++++++------------ database/rrdcalc.h | 115 ++-- database/rrdcalctemplate.c | 202 +++++- database/rrdcalctemplate.h | 35 +- database/rrdcontext.c | 188 +++--- database/rrdcontext.h | 2 + database/rrddim.c | 589 ++++++++++-------- database/rrddimvar.c | 278 +++++---- database/rrddimvar.h | 44 +- database/rrdfamily.c | 87 +-- database/rrdhost.c | 544 ++++++----------- database/rrdlabels.c | 97 +-- database/rrdset.c | 1085 +++++++++++++++++---------------- database/rrdsetvar.c | 305 ++++++--- database/rrdsetvar.h | 36 +- database/rrdvar.c | 272 +++++---- database/rrdvar.h | 74 ++- database/sqlite/sqlite_aclk.c | 3 - database/sqlite/sqlite_aclk_chart.c | 36 +- database/sqlite/sqlite_aclk_node.c | 5 +- database/sqlite/sqlite_db_migration.c | 18 + database/sqlite/sqlite_functions.c | 381 +----------- database/sqlite/sqlite_functions.h | 13 +- database/sqlite/sqlite_health.c | 15 +- 28 files changed, 2874 insertions(+), 2874 deletions(-) (limited to 'database') diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c index 50625d1e01..dc24bf32a5 100644 --- a/database/engine/journalfile.c +++ b/database/engine/journalfile.c @@ -519,7 +519,7 @@ int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfi journalfile->file = file; journalfile->pos = file_size; journalfile->data = netdata_mmap(path, file_size, MAP_SHARED, 0); - info("Loading journal file \"%s\" using %s.", path, journalfile->data?"using MMAP":"using uv_fs_read"); + info("Loading journal file \"%s\" using %s.", path, journalfile->data?"MMAP":"uv_fs_read"); max_id = iterate_transactions(ctx, journalfile); diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 5acab14d3b..27cf4ef059 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -138,7 +138,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta uuid_copy(rd->metric_uuid, multihost_legacy_uuid); if (unlikely(need_to_store && !ctx->tier)) - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); + (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); } struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle)); diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c index 139f997c93..74674dbdbd 100644 --- a/database/ram/rrddim_mem.c +++ b/database/ram/rrddim_mem.c @@ -42,7 +42,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) { struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle; RRDDIM *rd = ch->rd; - memset(rd->db, 0, rd->entries * sizeof(storage_number)); + memset(rd->db, 0, rd->rrdset->entries * sizeof(storage_number)); } int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) { diff --git a/database/rrd.h b/database/rrd.h index f473e308ac..b536bdeda5 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -16,20 +16,21 @@ typedef struct storage_metric_handle STORAGE_METRIC_HANDLE; typedef struct rrdhost RRDHOST; typedef struct rrddim RRDDIM; typedef struct rrdset RRDSET; -typedef struct rrdvar RRDVAR; -typedef struct rrdsetvar RRDSETVAR; -typedef struct rrddimvar RRDDIMVAR; typedef struct rrdcalc RRDCALC; typedef struct rrdcalctemplate RRDCALCTEMPLATE; typedef struct alarm_entry ALARM_ENTRY; typedef struct context_param CONTEXT_PARAM; +typedef struct rrdfamily_acquired RRDFAMILY_ACQUIRED; +typedef struct rrdvar_acquired RRDVAR_ACQUIRED; +typedef struct rrdsetvar_acquired RRDSETVAR_ACQUIRED; +typedef struct rrdcalc_acquired RRDCALC_ACQUIRED; + typedef void *ml_host_t; typedef void *ml_dimension_t; // forward declarations struct rrddim_tier; -struct rrdset_volatile; struct context_param; #ifdef ENABLE_DBENGINE @@ -75,11 +76,6 @@ struct context_param { uint8_t flags; }; -#define META_CHART_UPDATED 1 -#define META_PLUGIN_UPDATED 2 -#define META_MODULE_UPDATED 4 -#define META_CHART_ACTIVATED 8 - #define UPDATE_EVERY 1 #define UPDATE_EVERY_MAX 3600 @@ -122,7 +118,9 @@ typedef enum rrd_memory_mode { RRD_MEMORY_MODE_MAP = 2, RRD_MEMORY_MODE_SAVE = 3, RRD_MEMORY_MODE_ALLOC = 4, - RRD_MEMORY_MODE_DBENGINE = 5 + RRD_MEMORY_MODE_DBENGINE = 5, + + // this is 8-bit } RRD_MEMORY_MODE; #define RRD_MEMORY_MODE_NONE_NAME "none" @@ -145,7 +143,9 @@ typedef enum rrd_algorithm { RRD_ALGORITHM_ABSOLUTE = 0, RRD_ALGORITHM_INCREMENTAL = 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2, - RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3 + RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3, + + // this is 8-bit } RRD_ALGORITHM; #define RRD_ALGORITHM_ABSOLUTE_NAME "absolute" @@ -159,34 +159,42 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); // ---------------------------------------------------------------------------- // RRD FAMILY -struct rrdfamily { - STRING *family; - DICTIONARY *rrdvar_root_index; - - size_t use_count; -}; -typedef struct rrdfamily RRDFAMILY; +extern const RRDFAMILY_ACQUIRED *rrdfamily_add_and_acquire(RRDHOST *host, const char *id); +extern void rrdfamily_release(RRDHOST *host, const RRDFAMILY_ACQUIRED *rfa); +extern void rrdfamily_index_init(RRDHOST *host); +extern void rrdfamily_index_destroy(RRDHOST *host); +extern DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rf); // ---------------------------------------------------------------------------- -// flags -// use this for configuration flags, not for state control -// flags are set/unset in a manner that is not thread safe -// and may lead to missing information. +// flags & options +// options are permanent configuration options (no atomics to alter/access them) +typedef enum rrddim_options { + RRDDIM_OPTION_NONE = 0, + RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers + RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers + + // this is 8-bit +} RRDDIM_OPTIONS; + +#define rrddim_option_check(rd, flag) ((rd)->flags & (flag)) +#define rrddim_option_set(rd, flag) (rd)->flags |= (flag) +#define rrddim_option_clear(rd, flag) (rd)->flags &= ~(flag) + +// flags are runtime changing status flags (atomics are required to alter/access them) typedef enum rrddim_flags { RRDDIM_FLAG_NONE = 0, - RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers - RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers RRDDIM_FLAG_OBSOLETE = (1 << 2), // this is marked by the collector/module as obsolete - // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at + // No new values have been collected for this dimension since agent start, or it was marked RRDDIM_FLAG_OBSOLETE at // least rrdset_free_obsolete_time seconds ago. RRDDIM_FLAG_ARCHIVED = (1 << 3), RRDDIM_FLAG_ACLK = (1 << 4), - RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet + RRDDIM_FLAG_PENDING_FOREACH_ALARMS = (1 << 5), // set when foreach alarm has not been initialized yet RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database - RRDDIM_FLAG_INDEXED_ID = (1 << 7), + + // this is 8 bit } RRDDIM_FLAGS; #define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -231,6 +239,7 @@ extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src); void reload_host_labels(void); extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels); +extern void rrdset_save_rrdlabels_to_sql(RRDSET *st); extern int rrdlabels_unittest(void); @@ -244,13 +253,15 @@ struct rrddim { uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ - // the dimension definition + // dimension definition STRING *id; // the id of this dimension (for internal identification) STRING *name; // the name of this dimension (as presented to user) - RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension - RRDDIM_FLAGS flags; // configuration flags for the dimension + + RRD_ALGORITHM algorithm:8; // the algorithm that is applied to add new collected values + RRDDIM_OPTIONS options:8; // permanent configuration options + RRD_MEMORY_MODE rrd_memory_mode:8; // the memory mode for this dimension + /*RRDDIM_FLAGS*/ uint8_t flags; // run time changing status flags bool updated; // 1 when the dimension has been updated since the last processing bool exposed; // 1 when set what have sent this dimension to the central netdata @@ -258,57 +269,54 @@ struct rrddim { collected_number multiplier; // the multiplier of the collected values collected_number divisor; // the divider of the collected values - // ------------------------------------------------------------------------ - // members for temporary data we need for calculations + int update_every; // every how many seconds is this updated + // TODO - remove update_every from rrddim + // it is always the same in rrdset - struct timeval last_collected_time; // when was this dimension last updated - // this is actual date time we updated the last_collected_value - // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET + // ------------------------------------------------------------------------ + // operational state members #ifdef ENABLE_ACLK int aclk_live_status; #endif - ml_dimension_t ml_dimension; - - struct rrddim_tier *tiers[RRD_STORAGE_TIERS]; // our tiers of databases - - size_t collections_counter; // the number of times we added values to this rrddim - collected_number collected_value_max; // the absolute maximum of the collected value - - NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used - NETDATA_DOUBLE last_calculated_value; // the last calculated value processed - NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation) - collected_number collected_value; // the current value, as collected - resets to 0 after being used - collected_number last_collected_value; // the last value that was collected, after being processed + ml_dimension_t ml_dimension; // machine learning data about this dimension - // the *_volume members are used to calculate the accuracy of the rounding done by the - // storage number - they are printed to debug.log when debug is enabled for a set. - NETDATA_DOUBLE collected_volume; // the sum of all collected values so far - NETDATA_DOUBLE stored_volume; // the sum of all stored values so far + // ------------------------------------------------------------------------ + // linking to siblings and parents struct rrddim *next; // linking of dimensions within the same data set struct rrddim *prev; // linking of dimensions within the same data set struct rrdset *rrdset; + RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension // ------------------------------------------------------------------------ - // members for checking the data when loading from disk + // data collection members - long entries; // how many entries this dimension has in ram - // this is the same to the entries of the data set - // we set it here, to check the data when we load it from disk. + struct rrddim_tier *tiers[RRD_STORAGE_TIERS]; // our tiers of databases - int update_every; // every how many seconds is this updated + struct timeval last_collected_time; // when was this dimension last updated + // this is actual date time we updated the last_collected_value + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET - size_t memsize; // the memory allocated for this dimension (without RRDDIM) + size_t collections_counter; // the number of times we added values to this rrddim + collected_number collected_value_max; // the absolute maximum of the collected value + + NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used + NETDATA_DOUBLE last_calculated_value; // the last calculated value processed + NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation) - struct rrddimvar *variables; + collected_number collected_value; // the current value, as collected - resets to 0 after being used + collected_number last_collected_value; // the last value that was collected, after being processed // ------------------------------------------------------------------------ - // the values stored in this dimension, using our floating point numbers + // db mode RAM, SAVE, MAP, ALLOC, NONE specifics + // TODO - they should be managed by storage engine + // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) + size_t memsize; // the memory allocated for this dimension (without RRDDIM) void *rd_on_file; // pointer to the header written on disk storage_number *db; // the array of values }; @@ -346,7 +354,6 @@ struct rrddim_query_handle { RRDDIM *rd; time_t start_time; time_t end_time; - TIER_QUERY_FETCH tier_query_fetch_type; STORAGE_QUERY_HANDLE* handle; }; @@ -399,7 +406,7 @@ struct rrddim_collect_ops { // run this to flush / reset the current data collection sequence void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle); - // an finalization function to run after collection is over + // a finalization function to run after collection is over // returns 1 if it's safe to delete the dimension int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle); }; @@ -432,7 +439,6 @@ struct rrddim_query_ops { struct rrddim_tier { int tier_grouping; RRD_MEMORY_MODE mode; // the memory mode of this tier - RRD_BACKFILL backfill; // backfilling configuration STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle STORAGE_POINT virtual_point; @@ -448,11 +454,16 @@ extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t n // these loop macros make sure the linked list is accessed with the right lock #define rrddim_foreach_read(rd, st) \ - for((rd) = (st)->dimensions, rrdset_check_rdlock(st); (rd) ; (rd) = (rd)->next) + dfe_start_read((st)->rrddim_root_index, rd) #define rrddim_foreach_write(rd, st) \ - for((rd) = (st)->dimensions, rrdset_check_wrlock(st); (rd) ; (rd) = (rd)->next) + dfe_start_write((st)->rrddim_root_index, rd) + +#define rrddim_foreach_reentrant(rd, st) \ + dfe_start_reentrant((st)->rrddim_root_index, rd) +#define rrddim_foreach_done(rd) \ + dfe_done(rd) // ---------------------------------------------------------------------------- // RRDSET - this is a chart @@ -477,7 +488,7 @@ typedef enum rrdset_flags { RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions - // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at + // No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at // least rrdset_free_obsolete_time seconds ago. RRDSET_FLAG_ARCHIVED = (1 << 15), RRDSET_FLAG_ACLK = (1 << 16), @@ -496,14 +507,19 @@ typedef enum rrdset_flags { #define rrdset_is_ar_chart(st) rrdset_flag_check(st, RRDSET_FLAG_ANOMALY_RATE_CHART) struct rrdset { - uuid_t uuid; + uuid_t chart_uuid; // the global UUID for this chart // ------------------------------------------------------------------------ - // the set configuration + // chart configuration - STRING *id; // the ID of the data set - STRING *name; // the name of this dimension (as presented to user) - STRING *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + struct { + STRING *type; // the type of {type}.{id} + STRING *id; // the id of {type}.{id} + STRING *name; // the name of {type}.{name} + } parts; + + STRING *id; // the unique ID of the rrdset as {type}.{id} + STRING *name; // the unique name of the rrdset as {type}.{name} STRING *family; // grouping sets under the same family STRING *title; // title shown to user STRING *units; // units of measurement @@ -511,44 +527,51 @@ struct rrdset { STRING *plugin_name; // the name of the plugin that generated this STRING *module_name; // the name of the plugin module that generated this - RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart - RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to + RRDSET_TYPE chart_type; // line, area, stacked + + long priority; // the sorting priority of this chart + + int update_every; // data collection frequency + + DICTIONARY *rrdlabels; // chart labels + DICTIONARY *rrdsetvar_root_index; // chart variables + DICTIONARY *rrddimvar_root_index; // dimension variables + // we use this dictionary to manage their allocation + + // TODO - dimensions linked list and lock to be removed + netdata_rwlock_t rrdset_rwlock; // protects the dimensions linked list + RRDDIM *dimensions; // chart metrics + // ------------------------------------------------------------------------ + // operational state members + + RRDSET_FLAGS flags; // flags RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset - RRDSET_TYPE chart_type; // line, area, stacked - RRDSET_FLAGS flags; // configuration flags - RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances - int update_every; // every how many seconds is this updated? + uuid_t hash_uuid; // hash_id for syncing with cloud + // TODO - obsolete now - cleanup + + DICTIONARY *rrddim_root_index; // dimensions index int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored // netdata will interpolate values for gaps lower than this + // TODO - use the global - all charts have the same value - long entries; // total number of entries in the data set - - long current_entry; // the entry that is currently being updated - // it goes around in a round-robin fashion + // ------------------------------------------------------------------------ + // linking to siblings and parents - long priority; // the sorting priority of this chart + RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to + RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart + RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to // ------------------------------------------------------------------------ - // members for temporary data we need for calculations - - char *cache_dir; // the directory to store dimensions + // data collection members size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called - union { - time_t last_accessed_time; // the last time this RRDSET has been accessed - time_t last_entry_t; // the last_entry_t computed for transient RRDSET - }; - time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream - - uuid_t *chart_uuid; // Store the global GUID for this chart - // this object. - size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine + time_t last_accessed_time; // the last time this RRDSET has been accessed usec_t usec_since_last_update; // the time in microseconds since the last collection of data @@ -558,46 +581,73 @@ struct rrdset { total_number collected_total; // used internally to calculate percentages total_number last_collected_total; // used internally to calculate percentages - RRDFAMILY *rrdfamily; // pointer to RRDFAMILY this chart belongs to - RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to + size_t rrdlabels_last_saved_version; - struct rrdset *next; // linking of rrdsets - struct rrdset *prev; // linking of rrdsets + // ------------------------------------------------------------------------ + // data collection - streaming to parents, temp variables + + time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream // ------------------------------------------------------------------------ - // local variables + // context queries temp variables + // TODO - eliminate these - NETDATA_DOUBLE green; // green threshold for this chart - NETDATA_DOUBLE red; // red threshold for this chart + time_t last_entry_t; // the last_entry_t computed for transient RRDSET - DICTIONARY *rrdvar_root_index; // RRDVAR index for this chart - RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) - RRDCALC *alarms; // RRDCALC linked list for this chart + // ------------------------------------------------------------------------ + // dbengine specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) + + size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine // ------------------------------------------------------------------------ - // members for checking the data when loading from disk + // db mode SAVE, MAP specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) + char *cache_dir; // the directory to store dimensions unsigned long memsize; // how much mem we have allocated for this (without dimensions) void *st_on_file; // compatibility with V019 RRDSET files // ------------------------------------------------------------------------ - // chart labels + // db mode RAM, SAVE, MAP, ALLOC, NONE specifics + // TODO - they should be managed by storage engine + // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction) - DICTIONARY *rrdlabels; + long entries; // total number of entries in the data set + + long current_entry; // the entry that is currently being updated + // it goes around in a round-robin fashion // ------------------------------------------------------------------------ - // the dimensions + // exporting to 3rd party time-series members + // TODO - they should be managed by exporting engine + // (RRDSET_EXPORTING_STATE ptr to an undefined structure, and a call to clean this up during destruction) - DICTIONARY *rrddim_root_index; // the root of the dimensions index + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances + + // ------------------------------------------------------------------------ + // health monitoring members + // TODO - they should be managed by health + // (RRDSET_HEALTH_STATE ptr to an undefined structure, and a call to clean this up during destruction) - netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list - RRDDIM *dimensions; // the actual data for every dimension + NETDATA_DOUBLE green; // green threshold for this chart + NETDATA_DOUBLE red; // red threshold for this chart + + DICTIONARY *rrdvars; // RRDVAR index for this chart + const RRDFAMILY_ACQUIRED *rrdfamily; // pointer to RRDFAMILY dictionary item, this chart belongs to + + struct { + netdata_rwlock_t rwlock; // protection for RRDCALC *base + RRDCALC *base; // double linked list of RRDCALC related to this RRDSET + } alerts; }; #define rrdset_plugin_name(st) string2str((st)->plugin_name) #define rrdset_module_name(st) string2str((st)->module_name) #define rrdset_units(st) string2str((st)->units) -#define rrdset_type(st) string2str((st)->type) +#define rrdset_parts_type(st) string2str((st)->parts.type) #define rrdset_family(st) string2str((st)->family) #define rrdset_title(st) string2str((st)->title) #define rrdset_context(st) string2str((st)->context) @@ -614,11 +664,19 @@ extern STRING *rrd_string_strdupz(const char *s); // these loop macros make sure the linked list is accessed with the right lock #define rrdset_foreach_read(st, host) \ - for((st) = (host)->rrdset_root, rrdhost_check_rdlock(host); st ; (st) = (st)->next) + dfe_start_read((host)->rrdset_root_index, st) #define rrdset_foreach_write(st, host) \ - for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next) + dfe_start_write((host)->rrdset_root_index, st) + +#define rrdset_foreach_reentrant(st, host) \ + dfe_start_reentrant((host)->rrdset_root_index, st) +#define rrdset_foreach_done(st) \ + dfe_done(st) + +#define rrdset_number_of_dimensions(st) \ + dictionary_entries((st)->rrddim_root_index) extern void rrdset_memory_file_save(RRDSET *st); extern void rrdset_memory_file_free(RRDSET *st); @@ -633,19 +691,23 @@ extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) - RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts - RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan - RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases - RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet - RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms - RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), - RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates - RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid - RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname - RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent + RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) + RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts + RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan + RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases + RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet + RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms + RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), + RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid + RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname + RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent + RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 14), // the host has initialized health structures + RRDHOST_FLAG_INITIALIZED_RRDPUSH = (1 << 15), // the host has initialized rrdpush structures + RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 16), // the host has pending chart obsoletions + RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 17), // the host has pending dimension obsoletions } RRDHOST_FLAGS; #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -868,26 +930,17 @@ struct rrdhost { // all RRDCALCs are primarily allocated and linked here // RRDCALCs may be linked to charts at any point // (charts may or may not exist when these are loaded) - RRDCALC *host_alarms; - RRDCALC *alarms_with_foreach; - - ALARM_LOG health_log; // alarms historical events (event log) - uint32_t health_last_processed_id; // the last processed health id from the log - uint32_t health_max_unique_id; // the max alarm log unique id given for the host - uint32_t health_max_alarm_id; // the max alarm id given for the host + DICTIONARY *rrdcalc_root_index; // templates of alarms // these are used to create alarms when charts // are created or renamed, that match them - RRDCALCTEMPLATE *alarms_templates; - - // ------------------------------------------------------------------------ - // the charts of the host - - RRDSET *rrdset_root; // the host charts - - unsigned int obsolete_charts_count; + DICTIONARY *rrdcalctemplate_root_index; + ALARM_LOG health_log; // alarms historical events (event log) + uint32_t health_last_processed_id; // the last processed health id from the log + uint32_t health_max_unique_id; // the max alarm log unique id given for the host + uint32_t health_max_alarm_id; // the max alarm id given for the host // ------------------------------------------------------------------------ // locks @@ -909,7 +962,8 @@ struct rrdhost { DICTIONARY *rrdset_root_index_name; // the host's charts index (by name) DICTIONARY *rrdfamily_root_index; // the host's chart families index - DICTIONARY *rrdvar_root_index; // the host's chart variables index + DICTIONARY *rrdvars; // the host's chart variables index + // this includes custom host variables STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers @@ -973,6 +1027,11 @@ extern netdata_rwlock_t rrd_rwlock; // ---------------------------------------------------------------------------- extern bool is_storage_engine_shared(STORAGE_INSTANCE *engine); +extern void rrdset_index_init(RRDHOST *host); +extern void rrdset_index_destroy(RRDHOST *host); + +extern void rrddim_index_init(RRDSET *st); +extern void rrddim_index_destroy(RRDSET *st); // ---------------------------------------------------------------------------- @@ -1058,7 +1117,7 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns // ---------------------------------------------------------------------------- // RRDSET functions -extern int rrdset_set_name(RRDSET *st, const char *name); +extern int rrdset_reset_name(RRDSET *st, const char *name); extern RRDSET *rrdset_create_custom(RRDHOST *host , const char *type @@ -1086,12 +1145,10 @@ extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); extern void rrdhost_cleanup_all(void); -extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host); extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info); extern void rrdhost_free(RRDHOST *host, bool force); extern void rrdhost_save_charts(RRDHOST *host); extern void rrdhost_delete_charts(RRDHOST *host); -extern void rrd_cleanup_obsolete_charts(); extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now); @@ -1140,95 +1197,19 @@ extern void rrdset_is_obsolete(RRDSET *st); extern void rrdset_isnot_obsolete(RRDSET *st); // checks if the RRDSET should be offered to viewers -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) +#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) #define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) -// get the timestamp of the last entry in the round robin database -static inline time_t rrddim_last_entry_t(RRDDIM *rd) { - time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle); - - for(int tier = 1; tier < storage_tiers ;tier++) { - if(unlikely(!rd->tiers[tier])) continue; - - time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle); - if(t > latest) - latest = t; - } - - return latest; -} - -static inline time_t rrddim_first_entry_t(RRDDIM *rd) { - time_t oldest = 0; - - for(int tier = 0; tier < storage_tiers ;tier++) { - if(unlikely(!rd->tiers[tier])) continue; - - time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle); - if(t != 0 && (oldest == 0 || t < oldest)) - oldest = t; - } - - return oldest; -} - -// get the timestamp of the last entry in the round robin database -static inline time_t rrdset_last_entry_t_nolock(RRDSET *st) { - RRDDIM *rd; - time_t last_entry_t = 0; - - rrddim_foreach_read(rd, st) { - time_t t = rrddim_last_entry_t(rd); - if(t > last_entry_t) last_entry_t = t; - } - - return last_entry_t; -} - -static inline time_t rrdset_last_entry_t(RRDSET *st) { - time_t last_entry_t; - - netdata_rwlock_rdlock(&st->rrdset_rwlock); - last_entry_t = rrdset_last_entry_t_nolock(st); - netdata_rwlock_unlock(&st->rrdset_rwlock); - - return last_entry_t; -} - -// get the timestamp of first entry in the round robin database -static inline time_t rrdset_first_entry_t_nolock(RRDSET *st) { - RRDDIM *rd; - time_t first_entry_t = LONG_MAX; - - rrddim_foreach_read(rd, st) { - time_t t = rrddim_first_entry_t(rd); - if(t < first_entry_t) - first_entry_t = t; - } - - if (unlikely(LONG_MAX == first_entry_t)) return 0; - return first_entry_t; -} - -static inline time_t rrdset_first_entry_t(RRDSET *st) -{ - time_t first_entry_t; - - netdata_rwlock_rdlock(&st->rrdset_rwlock); - first_entry_t = rrdset_first_entry_t_nolock(st); - netdata_rwlock_unlock(&st->rrdset_rwlock); - - return first_entry_t; -} - -time_t rrdhost_last_entry_t(RRDHOST *h); +extern time_t rrddim_first_entry_t(RRDDIM *rd); +extern time_t rrddim_last_entry_t(RRDDIM *rd); +extern time_t rrdset_last_entry_t(RRDSET *st); +extern time_t rrdset_first_entry_t(RRDSET *st); +extern time_t rrdhost_last_entry_t(RRDHOST *h); // ---------------------------------------------------------------------------- // RRD DIMENSION functions -extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host); - extern RRDDIM *rrddim_add_custom(RRDSET *st , const char *id , const char *name @@ -1241,7 +1222,7 @@ extern RRDDIM *rrddim_add_custom(RRDSET *st #define rrddim_add(st, id, name, multiplier, divisor, algorithm) \ rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode) -extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); +extern int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name); extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm); extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier); extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor); @@ -1270,19 +1251,17 @@ extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); // ---------------------------------------------------------------------------- // RRD internal functions +extern void rrdset_delete_files(RRDSET *st); +extern void rrdset_save(RRDSET *st); +extern void rrdset_free(RRDSET *st); + #ifdef NETDATA_RRD_INTERNALS extern char *rrdset_cache_dir(RRDHOST *host, const char *id); extern void rrddim_free(RRDSET *st, RRDDIM *rd); -extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id); -extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); - -extern void rrdset_free(RRDSET *st); extern void rrdset_reset(RRDSET *st); -extern void rrdset_save(RRDSET *st); -extern void rrdset_delete_files(RRDSET *st); extern void rrdset_delete_obsolete_dimensions(RRDSET *st); extern RRDHOST *rrdhost_create( diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 83dd527325..bd92c7a700 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -1,10 +1,9 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_HEALTH_INTERNALS #include "rrd.h" // ---------------------------------------------------------------------------- -// RRDCALC management +// RRDCALC helpers inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { switch(status) { @@ -35,7 +34,38 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { } } -static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { +uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + + // re-use old IDs, by looking them up in the alarm log + ALARM_ENTRY *ae = NULL; + for(ae = host->health_log.alarms; ae ;ae = ae->next) { + if(unlikely(name == ae->name && chart == ae->chart)) { + if(next_event_id) *next_event_id = ae->alarm_event_id + 1; + break; + } + } + + uint32_t alarm_id; + + if(ae) + alarm_id = ae->alarm_id; + + else { + if (unlikely(!host->health_log.next_alarm_id)) + host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); + + alarm_id = host->health_log.next_alarm_id++; + } + + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock); + return alarm_id; +} + +// ---------------------------------------------------------------------------- +// RRDCALC replacing info text variables with RRDSET labels + +static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RRDCALC *rc) { if (!line || !*line) return NULL; @@ -85,19 +115,62 @@ static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { return ret; } -void rrdcalc_update_rrdlabels(RRDSET *st) { - RRDCALC *rc; - foreach_rrdcalc_in_rrdset(st, rc) { - if (rc->original_info) { - if (rc->info) - string_freez(rc->info); +void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc) { + if(!rc->rrdset || !rc->original_info || !rc->rrdset->rrdlabels) return; - rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); - } + size_t labels_version = dictionary_version(rc->rrdset->rrdlabels); + if(rc->labels_version != labels_version) { + + STRING *old = rc->info; + rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc); + string_freez(old); + + rc->labels_version = labels_version; } } -static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { +// ---------------------------------------------------------------------------- +// RRDCALC index management for RRDSET + +// the dictionary requires a unique key for every item +// we use {chart id}.{alert name} for both the RRDHOST and RRDSET alert indexes. + +#define RRDCALC_MAX_KEY_SIZE 1024 +static size_t rrdcalc_key(char *dst, size_t dst_len, const char *chart, const char *alert) { + return snprintfz(dst, dst_len, "%s/%s", chart, alert); +} + +const RRDCALC_ACQUIRED *rrdcalc_from_rrdset_get(RRDSET *st, const char *alert_name) { + char key[RRDCALC_MAX_KEY_SIZE + 1]; + size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_id(st), alert_name); + + const RRDCALC_ACQUIRED *rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1)); + + if(!rca) { + key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_name(st), alert_name); + rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1)); + } + + return rca; +} + +void rrdcalc_from_rrdset_release(RRDSET *st, const RRDCALC_ACQUIRED *rca) { + if(!rca) return; + + dictionary_acquired_item_release(st->rrdhost->rrdcalc_root_index, (const DICTIONARY_ITEM *)rca); +} + +RRDCALC *rrdcalc_acquired_to_rrdcalc(const RRDCALC_ACQUIRED *rca) { + if(rca) + return dictionary_acquired_item_value((const DICTIONARY_ITEM *)rca); + + return NULL; +} + +// ---------------------------------------------------------------------------- +// RRDCALC managing the linking with RRDSET + +static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) { RRDHOST *host = st->rrdhost; debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); @@ -105,7 +178,9 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { rc->last_status_change = now_realtime_sec(); rc->rrdset = st; - DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); + netdata_rwlock_wrlock(&st->alerts.rwlock); + DOUBLE_LINKED_LIST_APPEND_UNSAFE(st->alerts.base, rc, prev, next); + netdata_rwlock_unlock(&st->alerts.rwlock); if(rc->update_every < rc->rrdset->update_every) { error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); @@ -114,40 +189,61 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { if(!isnan(rc->green) && isnan(st->green)) { debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO - " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); + " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); st->green = rc->green; } if(!isnan(rc->red) && isnan(st->red)) { debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO - ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); + ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); st->red = rc->red; } - rc->local = rrdvar_create_and_index("local", st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); - rc->family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); + char buf[RRDVAR_MAX_LENGTH + 1]; + snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); + STRING *rrdset_name_rrdcalc_name = string_strdupz(buf); + snprintfz(buf, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); + STRING *rrdset_id_rrdcalc_name = string_strdupz(buf); - char fullname[RRDVAR_MAX_LENGTH + 1]; - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); - STRING *fullname_string = string_strdupz(fullname); - rc->hostid = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); - - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); - rc->hostname = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); - - string_freez(fullname_string); - - if(rc->hostid && !rc->hostname) - rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR; - - if(!rc->units) rc->units = string_dup(st->units); - - if (rc->original_info) { - if (rc->info) - string_freez(rc->info); + rc->rrdvar_local = rrdvar_add_and_acquire( + "local", + st->rrdvars, + rc->name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_LOCAL_VAR, + &rc->value); - rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); - } + rc->rrdvar_family = rrdvar_add_and_acquire( + "family", + rrdfamily_rrdvars_dict(st->rrdfamily), + rc->name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_FAMILY_VAR, + &rc->value); + + rc->rrdvar_host_chart_name = rrdvar_add_and_acquire( + "host", + host->rrdvars, + rrdset_name_rrdcalc_name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_HOST_CHARTNAME_VAR, + &rc->value); + + rc->rrdvar_host_chart_id = rrdvar_add_and_acquire( + "host", + host->rrdvars, + rrdset_id_rrdcalc_name, + RRDVAR_TYPE_CALCULATED, + RRDVAR_FLAG_RRDCALC_HOST_CHARTID_VAR | ((rc->rrdvar_host_chart_name) ? 0 : RRDVAR_FLAG_RRDCALC_HOST_CHARTNAME_VAR), + &rc->value); + + string_freez(rrdset_id_rrdcalc_name); + string_freez(rrdset_name_rrdcalc_name); + + if(!rc->units) + rc->units = string_dup(st->units); + + rrdcalc_update_info_using_rrdset_labels(rc); time_t now = now_realtime_sec(); @@ -177,43 +273,10 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { 0, rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); - health_alarm_log(host, ae); -} - -static inline int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { - if ( (rc->chart != st->id) - && (rc->chart != st->name)) - return 0; - - if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) - return 0; - - if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) - return 0; - - if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->rrdlabels, rc->host_labels_pattern, '=')) - return 0; - - return 1; -} - -// this has to be called while the RRDHOST is locked -inline void rrdsetcalc_link_matching(RRDSET *st) { - RRDHOST *host = st->rrdhost; - // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); - - RRDCALC *rc; - foreach_rrdcalc_in_rrdhost(host, rc) { - if(unlikely(rc->rrdset)) - continue; - - if(unlikely(rrdcalc_is_matching_rrdset(rc, st))) - rrdsetcalc_link(st, rc); - } + health_alarm_log_add_entry(host, ae); } -// this has to be called while the RRDHOST is locked -inline void rrdsetcalc_unlink(RRDCALC *rc) { +static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) { RRDSET *st = rc->rrdset; if(!st) { @@ -252,139 +315,176 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { 0, 0); - health_alarm_log(host, ae); + health_alarm_log_add_entry(host, ae); debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); // unlink it - DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); - - rrdvar_free(host, st->rrdvar_root_index, rc->local); - rc->local = NULL; - rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rc->family); - rc->family = NULL; + if(!having_ll_wrlock) + netdata_rwlock_wrlock(&st->alerts.rwlock); - rrdvar_free(host, host->rrdvar_root_index, rc->hostid); - rc->hostid = NULL; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alerts.base, rc, prev, next); - rrdvar_free(host, host->rrdvar_root_index, rc->hostname); - rc->hostname = NULL; + if(!having_ll_wrlock) + netdata_rwlock_unlock(&st->alerts.rwlock); rc->rrdset = NULL; + rrdvar_release_and_del(st->rrdvars, rc->rrdvar_local); + rc->rrdvar_local = NULL; + + rrdvar_release_and_del(rrdfamily_rrdvars_dict(st->rrdfamily), rc->rrdvar_family); + rc->rrdvar_family = NULL; + + rrdvar_release_and_del(host->rrdvars, rc->rrdvar_host_chart_id); + rc->rrdvar_host_chart_id = NULL; + + rrdvar_release_and_del(host->rrdvars, rc->rrdvar_host_chart_name); + rc->rrdvar_host_chart_name = NULL; + // RRDCALC will remain in RRDHOST // so that if the matching chart is found in the future // it will be applied automatically } -RRDCALC *rrdcalc_find(RRDSET *st, const char *name) { - RRDCALC *rc = NULL; +static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) { + if ( (rc->chart != st->id) + && (rc->chart != st->name)) + return false; - STRING *name_string = string_strdupz(name); + if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) + return false; - foreach_rrdcalc_in_rrdset(st, rc) { - if(unlikely(rc->name == name_string)) - break; - } + if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) + return false; - string_freez(name_string); + if (st->rrdhost->rrdlabels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->rrdlabels, rc->host_labels_pattern, '=')) + return false; - return rc; + return true; } -inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name) { +void rrdcalc_link_matching_alerts_to_rrdset(RRDSET *st) { + RRDHOST *host = st->rrdhost; + // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); + RRDCALC *rc; + foreach_rrdcalc_in_rrdhost_read(host, rc) { + if(rc->rrdset) + continue; - if(unlikely(!chart)) { - error("attempt to find RRDCALC '%s' without giving a chart name", name); - return 1; + if(unlikely(rrdcalc_check_if_it_matches_rrdset(rc, st))) + rrdcalc_link_to_rrdset(st, rc); } + foreach_rrdcalc_in_rrdhost_done(rc); +} - STRING *name_string = string_strdupz(name); - STRING *chart_string = string_strdupz(chart); - int ret = 0; +static inline int rrdcalc_check_and_link_rrdset_callback(RRDSET *st, void *rrdcalc) { + RRDCALC *rc = rrdcalc; - // make sure it does not already exist - foreach_rrdcalc_in_rrdhost(host, rc) { - if (unlikely(rc->chart == chart_string && rc->name == name_string)) { - debug(D_HEALTH, "Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); - info("Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host));