From 5e1b95cf92168c4df74586fb4430dc284806da82 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Mon, 5 Sep 2022 19:31:06 +0300 Subject: Deduplicate all netdata strings (#13570) * rrdfamily * rrddim * rrdset plugin and module names * rrdset units * rrdset type * rrdset family * rrdset title * rrdset title more * rrdset context * rrdcalctemplate context and removal of context hash from rrdset * strings statistics * rrdset name * rearranged members of rrdset * eliminate rrdset name hash; rrdcalc chart converted to STRING * rrdset id, eliminated rrdset hash * rrdcalc, alarm_entry, alert_config and some of rrdcalctemplate * rrdcalctemplate * rrdvar * eval_variable * rrddimvar and rrdsetvar * rrdhost hostname, os and tags * fix master commits * added thread cache; implemented string_dup without locks * faster thread cache * rrdset and rrddim now use dictionaries for indexing * rrdhost now uses dictionary * rrdfamily now uses DICTIONARY * rrdvar using dictionary instead of AVL * allocate the right size to rrdvar flag members * rrdhost remaining char * members to STRING * * better error handling on indexing * strings now use a read/write lock to allow parallel searches to the index * removed AVL support from dictionaries; implemented STRING with native Judy calls * string releases should be negative * only 31 bits are allowed for enum flags * proper locking on strings * string threading unittest and fixes * fix lgtm finding * fixed naming * stream chart/dimension definitions at the beginning of a streaming session * thread stack variable is undefined on thread cancel * rrdcontext garbage collect per host on startup * worker control in garbage collection * relaxed deletion of rrdmetrics * type checking on dictfe * netdata chart to monitor rrdcontext triggers * Group chart label updates * rrdcontext better handling of collected rrdsets * rrdpush incremental transmition of definitions should use as much buffer as possible * require 1MB per chart * empty the sender buffer before enabling metrics streaming * fill up to 50% of buffer * reset signaling metrics sending * use the shared variable for status * use separate host flag for enabling streaming of metrics * make sure the flag is clear * add logging for streaming * add logging for streaming on buffer overflow * circular_buffer proper sizing * removed obsolete logs * do not execute worker jobs if not necessary * better messages about compression disabling * proper use of flags and updating rrdset last access time every time the obsoletion flag is flipped * monitor stream sender used buffer ratio * Update exporting unit tests * no need to compare label value with strcmp * streaming send workers now monitor bandwidth * workers now use strings * streaming receiver monitors incoming bandwidth * parser shift of worker ids * minor fixes * Group chart label updates * Populate context with dimensions that have data * Fix chart id * better shift of parser worker ids * fix for streaming compression * properly count received bytes * ensure LZ4 compression ring buffer does not wrap prematurely * do not stream empty charts; do not process empty instances in rrdcontext * need_to_send_chart_definition() does not need an rrdset lock any more * rrdcontext objects are collected, after data have been written to the db * better logging of RRDCONTEXT transitions * always set all variables needed by the worker utilization charts * implemented double linked list for most objects; eliminated alarm indexes from rrdhost; and many more fixes * lockless strings design - string_dup() and string_freez() are totally lockless when they dont need to touch Judy - only Judy is protected with a read/write lock * STRING code re-organization for clarity * thread_cache improvements; double numbers precision on worker threads * STRING_ENTRY now shadown STRING, so no duplicate definition is required; string_length() renamed to string_strlen() to follow the paradigm of all other functions, STRING internal statistics are now only compiled with NETDATA_INTERNAL_CHECKS * rrdhost index by hostname now cleans up; aclk queries of archieved hosts do not index hosts * Add index to speed up database context searches * Removed last_updated optimization (was also buggy after latest merge with master) Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> Co-authored-by: Vladimir Kobal --- database/engine/metadata_log/metalogpluginsd.c | 4 +- database/engine/rrdengineapi.c | 5 +- database/ram/rrddim_mem.c | 8 +- database/rrd.c | 12 + database/rrd.h | 349 +++++++++---------- database/rrdcalc.c | 454 +++++++++---------------- database/rrdcalc.h | 142 ++++---- database/rrdcalctemplate.c | 75 ++-- database/rrdcalctemplate.h | 58 ++-- database/rrdcontext.c | 444 ++++++++++++------------ database/rrddim.c | 183 +++++----- database/rrddimvar.c | 119 +++---- database/rrddimvar.h | 23 +- database/rrdfamily.c | 46 +-- database/rrdhost.c | 425 +++++++++++------------ database/rrdlabels.c | 24 +- database/rrdset.c | 449 +++++++++++------------- database/rrdsetvar.c | 74 ++-- database/rrdsetvar.h | 8 +- database/rrdvar.c | 150 ++++---- database/rrdvar.h | 27 +- database/sqlite/sqlite_aclk.c | 26 +- database/sqlite/sqlite_aclk.h | 16 +- database/sqlite/sqlite_aclk_alert.c | 56 ++- database/sqlite/sqlite_aclk_chart.c | 55 ++- database/sqlite/sqlite_aclk_node.c | 16 +- database/sqlite/sqlite_context.c | 1 - database/sqlite/sqlite_functions.c | 169 +++++---- database/sqlite/sqlite_health.c | 195 +++++------ 29 files changed, 1698 insertions(+), 1915 deletions(-) (limited to 'database') diff --git a/database/engine/metadata_log/metalogpluginsd.c b/database/engine/metadata_log/metalogpluginsd.c index a5301bc108..dcf2deb7d2 100755 --- a/database/engine/metadata_log/metalogpluginsd.c +++ b/database/engine/metadata_log/metalogpluginsd.c @@ -12,11 +12,11 @@ PARSER_RC metalog_pluginsd_host_action( { struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private; - RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0); + RRDHOST *host = rrdhost_find_by_guid(machine_guid); if (host) { if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) { error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.", - host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE)); ((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */ } diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index f4da294079..5acab14d3b 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -98,7 +98,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta pg_cache = &ctx->pg_cache; - rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid); + rrdeng_generate_legacy_uuid(rrddim_id(rd), (char *)rrdset_id(rd->rrdset), &legacy_uuid); if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx)) is_multihost_child = 1; @@ -138,8 +138,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_insta uuid_copy(rd->metric_uuid, multihost_legacy_uuid); if (unlikely(need_to_store && !ctx->tier)) - (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, - rd->algorithm); + (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm); } struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle)); diff --git a/database/ram/rrddim_mem.c b/database/ram/rrddim_mem.c index 3226d3c0de..139f997c93 100644 --- a/database/ram/rrddim_mem.c +++ b/database/ram/rrddim_mem.c @@ -91,7 +91,7 @@ static inline size_t rrddim_time2slot(RRDDIM *rd, time_t t) { } if(unlikely(ret >= entries)) { - error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rd->name); + error("INTERNAL ERROR: rrddim_time2slot() on %s returns values outside entries", rrddim_name(rd)); ret = entries - 1; } @@ -119,12 +119,12 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) { ret = last_entry_t - (time_t)(update_every * (last_slot - slot)); if(unlikely(ret < first_entry_t)) { - error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rd->name); + error("INTERNAL ERROR: rrddim_slot2time() on %s returns time too far in the past", rrddim_name(rd)); ret = first_entry_t; } if(unlikely(ret > last_entry_t)) { - error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rd->name); + error("INTERNAL ERROR: rrddim_slot2time() on %s returns time into the future", rrddim_name(rd)); ret = last_entry_t; } @@ -206,7 +206,7 @@ int rrddim_query_is_finished(struct rrddim_query_handle *handle) { void rrddim_query_finalize(struct rrddim_query_handle *handle) { #ifdef NETDATA_INTERNAL_CHECKS if(!rrddim_query_is_finished(handle)) - error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", handle->rd->rrdset->id, handle->rd->name); + error("QUERY: query for chart '%s' dimension '%s' has been stopped unfinished", rrdset_id(handle->rd->rrdset), rrddim_name(handle->rd)); #endif freez(handle->handle); } diff --git a/database/rrd.c b/database/rrd.c index f91039ea57..df364419ea 100644 --- a/database/rrd.c +++ b/database/rrd.c @@ -154,3 +154,15 @@ char *rrdset_cache_dir(RRDHOST *host, const char *id) { return ret; } +// ---------------------------------------------------------------------------- +// RRD - string management + +STRING *rrd_string_strdupz(const char *s) { + if(unlikely(!s || !*s)) return string_strdupz(s); + + char *tmp = strdupz(s); + json_fix_string(tmp); + STRING *ret = string_strdupz(tmp); + freez(tmp); + return ret; +} diff --git a/database/rrd.h b/database/rrd.h index f8919e83a5..36c4e5d58f 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -160,14 +160,10 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); // RRD FAMILY struct rrdfamily { - avl_t avl; - - const char *family; - uint32_t hash_family; + STRING *family; + DICTIONARY *rrdvar_root_index; size_t use_count; - - avl_tree_lock rrdvar_root_index; }; typedef struct rrdfamily RRDFAMILY; @@ -190,6 +186,7 @@ typedef enum rrddim_flags { RRDDIM_FLAG_PENDING_FOREACH_ALARM = (1 << 5), // set when foreach alarm has not been initialized yet RRDDIM_FLAG_META_HIDDEN = (1 << 6), // Status of hidden option in the metadata database + RRDDIM_FLAG_INDEXED_ID = (1 << 7), } RRDDIM_FLAGS; #define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & (flag)) @@ -244,27 +241,13 @@ extern bool exporting_labels_filter_callback(const char *name, const char *value // RRD DIMENSION - this is a metric struct rrddim { - // ------------------------------------------------------------------------ - // binary indexing structures - - avl_t avl; // the binary index - this has to be first member! - uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ // the dimension definition - const char *id; // the id of this dimension (for internal identification) - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) - uint32_t hash; // a simple hash of the id, to speed up searching / indexing - // instead of strcmp() every item in the binary index - // we first compare the hashes - uint32_t hash_name; // a simple hash of the name - - + STRING *id; // the id of this dimension (for internal identification) + STRING *name; // the name of this dimension (as presented to user) RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension RRDDIM_FLAGS flags; // configuration flags for the dimension @@ -305,6 +288,8 @@ struct rrddim { NETDATA_DOUBLE stored_volume; // the sum of all stored values so far struct rrddim *next; // linking of dimensions within the same data set + struct rrddim *prev; // linking of dimensions within the same data set + struct rrdset *rrdset; RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension @@ -328,6 +313,9 @@ struct rrddim { storage_number *db; // the array of values }; +#define rrddim_id(rd) string2str((rd)->id) +#define rrddim_name(rd) string2str((rd) ->name) + // returns the RRDDIM cache filename, or NULL if it does not exist extern const char *rrddim_cache_filename(RRDDIM *rd); @@ -459,9 +447,6 @@ extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t n // ---------------------------------------------------------------------------- // volatile state per chart struct rrdset_volatile { - char *old_title; - char *old_units; - char *old_context; uuid_t hash_id; DICTIONARY *chart_labels; bool is_ar_chart; @@ -485,88 +470,75 @@ struct rrdset_volatile { // and may lead to missing information. typedef enum rrdset_flags { - RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart - RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete - RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API and external databases - RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API and external databases - RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming) - RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming) - RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming) - RRDSET_FLAG_STORE_FIRST = 1 << 9, // if set, do not eliminate the first collection during interpolation - RRDSET_FLAG_HETEROGENEOUS = 1 << 10, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) - RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous - RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for exporting - RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now) - RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions - // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at - // least rrdset_free_obsolete_time seconds ago. - RRDSET_FLAG_ARCHIVED = 1 << 15, - RRDSET_FLAG_ACLK = 1 << 16, - RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms - RRDSET_FLAG_ANOMALY_DETECTION = 1 << 18 // flag to identify anomaly detection charts. + RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) + RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete + RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases + RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases + RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming) + RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming) + RRDSET_FLAG_UPSTREAM_EXPOSED = (1 << 8), // if set, we have sent this chart definition to netdata parent (streaming) + RRDSET_FLAG_STORE_FIRST = (1 << 9), // if set, do not eliminate the first collection during interpolation + RRDSET_FLAG_HETEROGENEOUS = (1 << 10), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 11), // if set, the chart should be checked to determine if the dimensions are homogeneous + RRDSET_FLAG_HIDDEN = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting + RRDSET_FLAG_SYNC_CLOCK = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions + // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at + // least rrdset_free_obsolete_time seconds ago. + RRDSET_FLAG_ARCHIVED = (1 << 15), + RRDSET_FLAG_ACLK = (1 << 16), + RRDSET_FLAG_PENDING_FOREACH_ALARMS = (1 << 17), // contains dims with uninitialized foreach alarms + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts. + RRDSET_FLAG_INDEXED_ID = (1 << 19), + RRDSET_FLAG_INDEXED_NAME = (1 << 20), } RRDSET_FLAGS; #define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & (flag)) #define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST) -#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~flag, __ATOMIC_SEQ_CST) +#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~(flag), __ATOMIC_SEQ_CST) struct rrdset { - // ------------------------------------------------------------------------ - // binary indexing structures - - avl_t avl; // the index, with key the id - this has to be first! - avl_t avlname; // the index, with key the name - // ------------------------------------------------------------------------ // the set configuration - char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set - - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) - - char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) - char *family; // grouping sets under the same family - char *title; // title shown to user - char *units; // units of measurement - - char *context; // the template of this data set - uint32_t hash_context; // the hash of the chart's context + STRING *id; // the ID of the data set + STRING *name; // the name of this dimension (as presented to user) + STRING *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + STRING *family; // grouping sets under the same family + STRING *title; // title shown to user + STRING *units; // units of measurement + STRING *context; // the template of this data set + STRING *plugin_name; // the name of the plugin that generated this + STRING *module_name; // the name of the plugin module that generated this RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to + RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset RRDSET_TYPE chart_type; // line, area, stacked + RRDSET_FLAGS flags; // configuration flags + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances int update_every; // every how many seconds is this updated? + int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this + long entries; // total number of entries in the data set long current_entry; // the entry that is currently being updated // it goes around in a round-robin fashion - RRDSET_FLAGS flags; // configuration flags - RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances - - int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored - // netdata will interpolate values for gaps lower than this - long priority; // the sorting priority of this chart // ------------------------------------------------------------------------ // members for temporary data we need for calculations - RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped - char *cache_dir; // the directory to store dimensions - netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list - size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called @@ -576,19 +548,12 @@ struct rrdset { }; time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream - char *plugin_name; // the name of the plugin that generated this - char *module_name; // the name of the plugin module that generated this uuid_t *chart_uuid; // Store the global GUID for this chart // this object. struct rrdset_volatile *state; // volatile state that is not persistently stored size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine - uint32_t hash; // a simple hash on the id, to speed up searching - // we first compare hashes, and only if the hashes are equal we do string comparisons - - uint32_t hash_name; // a simple hash on the name - usec_t usec_since_last_update; // the time in microseconds since the last collection of data struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) @@ -601,14 +566,15 @@ struct rrdset { RRDHOST *rrdhost; // pointer to RRDHOST this chart belongs to struct rrdset *next; // linking of rrdsets + struct rrdset *prev; // linking of rrdsets // ------------------------------------------------------------------------ // local variables - NETDATA_DOUBLE green; // green threshold for this chart - NETDATA_DOUBLE red; // red threshold for this chart + NETDATA_DOUBLE green; // green threshold for this chart + NETDATA_DOUBLE red; // red threshold for this chart - avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart + DICTIONARY *rrdvar_root_index; // RRDVAR index for this chart RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) RRDCALC *alarms; // RRDCALC linked list for this chart @@ -621,14 +587,27 @@ struct rrdset { // ------------------------------------------------------------------------ // the dimensions - avl_tree_lock dimensions_index; // the root of the dimensions index + DICTIONARY *rrddim_root_index; // the root of the dimensions index + + netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list RRDDIM *dimensions; // the actual data for every dimension }; +#define rrdset_plugin_name(st) string2str((st)->plugin_name) +#define rrdset_module_name(st) string2str((st)->module_name) +#define rrdset_units(st) string2str((st)->units) +#define rrdset_type(st) string2str((st)->type) +#define rrdset_family(st) string2str((st)->family) +#define rrdset_title(st) string2str((st)->title) +#define rrdset_context(st) string2str((st)->context) +#define rrdset_name(st) string2str((st)->name) +#define rrdset_id(st) string2str((st)->id) + #define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock)) #define rrdset_wrlock(st) netdata_rwlock_wrlock(&((st)->rrdset_rwlock)) #define rrdset_unlock(st) netdata_rwlock_unlock(&((st)->rrdset_rwlock)) +extern STRING *rrd_string_strdupz(const char *s); // ---------------------------------------------------------------------------- // these loop macros make sure the linked list is accessed with the right lock @@ -653,26 +632,29 @@ extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) - RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts - RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan - RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases - RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet - RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb - RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms - RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), - RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data) + RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts + RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan + RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases + RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet + RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb + RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms + RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8), + RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9), + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates + RRDHOST_FLAG_INDEXED_MACHINE_GUID = (1 << 11), // when set, we have indexed its machine guid + RRDHOST_FLAG_INDEXED_HOSTNAME = (1 << 12), // when set, we have indexed its hostname + RRDHOST_FLAG_STREAM_COLLECTED_METRICS = (1 << 13), // when set, rrdset_done() should push metrics to parent } RRDHOST_FLAGS; #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag)) #define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST) -#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~flag, __ATOMIC_SEQ_CST) +#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~(flag), __ATOMIC_SEQ_CST) #ifdef NETDATA_INTERNAL_CHECKS #define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ - debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0) + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0) #else #define rrdset_debug(st, fmt, args...) debug_dummy() #endif @@ -690,34 +672,30 @@ struct alarm_entry { time_t duration; time_t non_clear_duration; - char *name; - uint32_t hash_name; - - char *chart; - uint32_t hash_chart; - char *chart_context; + STRING *name; + STRING *chart; + STRING *chart_context; + STRING *family; - char *family; + STRING *classification; + STRING *component; + STRING *type; - char *classification; - char *component; - char *type; - - char *exec; - char *recipient; + STRING *exec; + STRING *recipient; time_t exec_run_timestamp; int exec_code; uint64_t exec_spawn_serial; - char *source; - char *units; - char *info; + STRING *source; + STRING *units; + STRING *info; NETDATA_DOUBLE old_value; NETDATA_DOUBLE new_value; - char *old_value_string; - char *new_value_string; + STRING *old_value_string; + STRING *new_value_string; RRDCALC_STATUS old_status; RRDCALC_STATUS new_status; @@ -737,6 +715,20 @@ struct alarm_entry { struct alarm_entry *prev_in_progress; }; +#define ae_name(ae) string2str((ae)->name) +#define ae_chart_name(ae) string2str((ae)->chart) +#define ae_chart_context(ae) string2str((ae)->chart_context) +#define ae_family(ae) string2str((ae)->family) +#define ae_classification(ae) string2str((ae)->classification) +#define ae_component(ae) string2str((ae)->component) +#define ae_type(ae) string2str((ae)->type) +#define ae_exec(ae) string2str((ae)->exec) +#define ae_recipient(ae) string2str((ae)->recipient) +#define ae_source(ae) string2str((ae)->source) +#define ae_units(ae) string2str((ae)->units) +#define ae_info(ae) string2str((ae)->info) +#define ae_old_value_string(ae) string2str((ae)->old_value_string) +#define ae_new_value_string(ae) string2str((ae)->new_value_string) typedef struct alarm_log { uint32_t next_log_id; @@ -790,24 +782,20 @@ struct rrdhost_system_info { }; struct rrdhost { - avl_t avl; // the index of hosts + char machine_guid[GUID_LEN + 1]; // the unique ID of this host // ------------------------------------------------------------------------ // host information - char *hostname; // the hostname of this host - uint32_t hash_hostname; // the hostname hash + STRING *hostname; // the hostname of this host + STRING *registry_hostname; // the registry hostname for this host + STRING *os; // the O/S type of the host + STRING *tags; // tags for this host + STRING *timezone; // the timezone of the host + STRING *abbrev_timezone; // the abbriviated timezone of the host + STRING *program_name; // the program name that collects metrics for this host + STRING *program_version; // the program version that collects metrics for this host - char *registry_hostname; // the registry hostname for this host - - char machine_guid[GUID_LEN + 1]; // the unique ID of this host - uint32_t hash_machine_guid; // the hash of the unique ID - - const char *os; // the O/S type of the host - const char *tags; // tags for this host - const char *timezone; // the timezone of the host - - const char *abbrev_timezone; // the abbriviated timezone of the host int32_t utc_offset; // the offset in seconds from utc RRDHOST_FLAGS flags; // flags about this RRDHOST @@ -820,9 +808,6 @@ struct rrdhost { char *cache_dir; // the directory to save RRD cache files char *varlib_dir; // the directory to save health log - char *program_name; // the program name that collects metrics for this host - char *program_version; // the program version that collects metrics for this host - struct rrdhost_system_info *system_info; // information collected from the host environment // ------------------------------------------------------------------------ @@ -873,23 +858,21 @@ struct rrdhost { // health monitoring options unsigned int health_enabled; // 1 when this host has health enabled - time_t health_delay_up_to; // a timestamp to delay alarms processing up to - char *health_default_exec; // the full path of the alarms notifications program - char *health_default_recipient; // the default recipient for all alarms - char *health_log_filename; // the alarms event log filename - size_t health_log_entries_written; // the number of alarm events written to the alarms event log - FILE *health_log_fp; // the FILE pointer to the open alarms event log file - uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications - uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications + time_t health_delay_up_to; // a timestamp to delay alarms processing up to + STRING *health_default_exec; // the full path of the alarms notifications program + STRING *health_default_recipient; // the default recipient for all alarms + char *health_log_filename; // the alarms event log filename + size_t health_log_entries_written; // the number of alarm events written to the alarms event log + FILE *health_log_fp; // the FILE pointer to the open alarms event log file + uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications + uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications // all RRDCALCs are primarily allocated and linked here // RRDCALCs may be linked to charts at any point // (charts may or may not exist when these are loaded) - RRDCALC *alarms; + RRDCALC *host_alarms; RRDCALC *alarms_with_foreach; - avl_tree_lock alarms_idx_health_log; - avl_tree_lock alarms_idx_name; ALARM_LOG health_log; // alarms historical events (event log) uint32_t health_last_processed_id; // the last processed health id from the log @@ -899,9 +882,7 @@ struct rrdhost { // templates of alarms // these are used to create alarms when charts // are created or renamed, that match them - RRDCALCTEMPLATE *templates; - RRDCALCTEMPLATE *alarms_template_with_foreach; - + RRDCALCTEMPLATE *alarms_templates; // ------------------------------------------------------------------------ // the charts of the host @@ -927,11 +908,11 @@ struct rrdhost { // ------------------------------------------------------------------------ // indexes - avl_tree_lock rrdset_root_index; // the host's charts index (by id) - avl_tree_lock rrdset_root_index_name; // the host's charts index (by name) + DICTIONARY *rrdset_root_index; // the host's charts index (by id) + DICTIONARY *rrdset_root_index_name; // the host's charts index (by name) - avl_tree_lock rrdfamily_root_index; // the host's chart families index - avl_tree_lock rrdvar_root_index; // the host's chart variables index + DICTIONARY *rrdfamily_root_index; // the host's chart families index + DICTIONARY *rrdvar_root_index; // the host's chart variables index STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers @@ -950,9 +931,19 @@ struct rrdhost { aclk_rrdhost_state aclk_state; struct rrdhost *next; + struct rrdhost *prev; }; extern RRDHOST *localhost; +#define rrdhost_hostname(host) string2str((host)->hostname) +#define rrdhost_registry_hostname(host) string2str((host)->registry_hostname) +#define rrdhost_os(host) string2str((host)->os) +#define rrdhost_tags(host) string2str((host)->tags) +#define rrdhost_timezone(host) string2str((host)->timezone) +#define rrdhost_abbrev_timezone(host) string2str((host)->abbrev_timezone) +#define rrdhost_program_name(host) string2str((host)->program_name) +#define rrdhost_program_version(host) string2str((host)->program_version) + #define rrdhost_rdlock(host) netdata_rwlock_rdlock(&((host)->rrdhost_rwlock)) #define rrdhost_wrlock(host) netdata_rwlock_wrlock(&((host)->rrdhost_rwlock)) #define rrdhost_unlock(host) netdata_rwlock_unlock(&((host)->rrdhost_rwlock)) @@ -960,6 +951,8 @@ extern RRDHOST *localhost; #define rrdhost_aclk_state_lock(host) netdata_mutex_lock(&((host)->aclk_state_lock)) #define rrdhost_aclk_state_unlock(host) netdata_mutex_unlock(&((host)->aclk_state_lock)) +extern long rrdhost_hosts_available(void); + // ---------------------------------------------------------------------------- // these loop macros make sure the linked list is accessed with the right lock @@ -990,8 +983,8 @@ extern time_t rrdhost_free_orphan_time; extern int rrd_init(char *hostname, struct rrdhost_system_info *system_info); -extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash); -extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash); +extern RRDHOST *rrdhost_find_by_hostname(const char *hostname); +extern RRDHOST *rrdhost_find_by_guid(const char *guid); extern RRDHOST *rrdhost_find_or_create( const char *hostname @@ -1237,11 +1230,18 @@ time_t rrdhost_last_entry_t(RRDHOST *h); // RRD DIMENSION functions extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host); -extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, - collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);//, - //int is_archived, uuid_t *dim_uuid); -#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \ - algorithm, (st)->rrd_memory_mode)//, 0, NULL) + +extern RRDDIM *rrddim_add_custom(RRDSET *st + , const char *id + , const char *name + , collected_number multiplier + , collected_number divisor + , RRD_ALGORITHM algorithm + , RRD_MEMORY_MODE memory_mode + ); + +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) \ + rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode) extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm); @@ -1249,15 +1249,7 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor); extern RRDDIM *rrddim_find(RRDSET *st, const char *id); -/* This will not return dimensions that are archived */ -static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id) -{ - RRDDIM *rd = rrddim_find(st, id); - if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) - return NULL; - return rd; -} - +extern RRDDIM *rrddim_find_active(RRDSET *st, const char *id); extern int rrddim_hide(RRDSET *st, const char *id); extern int rrddim_unhide(RRDSET *st, const char *id); @@ -1275,33 +1267,20 @@ extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries); // ---------------------------------------------------------------------------- // Miscellaneous functions -extern int alarm_compare_id(void *a, void *b); -extern int alarm_compare_name(void *a, void *b); +extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); // ---------------------------------------------------------------------------- // RRD internal functions #ifdef NETDATA_RRD_INTERNALS -extern avl_tree_lock rrdhost_root_index; - -extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); extern char *rrdset_cache_dir(RRDHOST *host, const char *id); extern void rrddim_free(RRDSET *st, RRDDIM *rd); -extern int rrddim_compare(void* a, void* b); -extern int rrdset_compare(void* a, void* b); -extern int rrdset_compare_name(void* a, void* b); -extern int rrdfamily_compare(void *a, void *b); - extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id); extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); -#define rrdset_index_add(host, st) (RRDSET *)avl_insert_lock(&((host)->rrdset_root_index), (avl_t *)(st)) -#define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl_t *)(st)) -extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st); - extern void rrdset_free(RRDSET *st); extern void rrdset_reset(RRDSET *st); extern void rrdset_save(RRDSET *st); @@ -1318,8 +1297,8 @@ extern RRDHOST *rrdhost_create( #endif /* NETDATA_RRD_INTERNALS */ extern void set_host_properties( - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *hostname, const char *registry_hostname, - const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, + RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname, + const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name, const char *program_version); extern int get_tier_grouping(int tier); diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 39fb09681a..808e74952a 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -35,9 +35,8 @@ inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { } } -char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) -{ - if (!line) +static STRING *rrdcalc_replace_variables(const char *line, RRDCALC *rc) { + if (!line || !*line) return NULL; size_t pos = 0; @@ -46,24 +45,28 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) char *m, *lbl_value = NULL; while ((m = strchr(temp + pos, '$'))) { - int i=0; + int i = 0; char *e = m; while (*e) { - if (*e == ' ' || i == RRDCALC_VAR_MAX - 1) { + + if (*e == ' ' || i == RRDCALC_VAR_MAX - 1) break; - } else - var[i]=*e; + var[i] = *e; + e++; i++; } - var[i]='\0'; + + var[i] = '\0'; pos = m - temp + 1; + if (!strcmp(var, RRDCALC_VAR_FAMILY)) { - char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rc->rrdset->family : "", m); + char *buf = find_and_replace(temp, var, (rc->rrdset && rc->rrdset->family) ? rrdset_family(rc->rrdset) : "", m); freez(temp); temp = buf; - } else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) { + } + else if (!strncmp(var, RRDCALC_VAR_LABEL, RRDCALC_VAR_LABEL_LEN)) { if(likely(rc->rrdset && rc->rrdset->state && rc->rrdset->state->chart_labels)) { rrdlabels_get_value_to_char_or_null(rc->rrdset->state->chart_labels, &lbl_value, var+RRDCALC_VAR_LABEL_LEN); if (lbl_value) { @@ -76,17 +79,20 @@ char *rrdcalc_replace_variables(const char *line, RRDCALC *rc) } } - return temp; + STRING *ret = string_strdupz(temp); + freez(temp); + + return ret; } void rrdcalc_update_rrdlabels(RRDSET *st) { RRDCALC *rc; - for( rc = st->alarms; rc ; rc = rc->rrdset_next ) { + foreach_rrdcalc_in_rrdset(st, rc) { if (rc->original_info) { if (rc->info) - freez(rc->info); + string_freez(rc->info); - rc->info = rrdcalc_replace_variables(rc->original_info, rc); + rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); } } } @@ -94,58 +100,57 @@ void rrdcalc_update_rrdlabels(RRDSET *st) { static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { RRDHOST *host = st->rrdhost; - debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname); + debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); rc->last_status_change = now_realtime_sec(); rc->rrdset = st; - rc->rrdset_next = st->alarms; - rc->rrdset_prev = NULL; - - if(rc->rrdset_next) - rc->rrdset_next->rrdset_prev = rc; - - st->alarms = rc; + DOUBLE_LINKED_LIST_PREPEND_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); if(rc->update_every < rc->rrdset->update_every) { - error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); + error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every); rc->update_every = rc->rrdset->update_every; } if(!isnan(rc->green) && isnan(st->green)) { debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO - " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green); + " to " NETDATA_DOUBLE_FORMAT_AUTO ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->green, rc->green); st->green = rc->green; } if(!isnan(rc->red) && isnan(st->red)) { debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO - ".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red); + ".", rrdset_id(rc->rrdset), rrdcalc_name(rc), rc->rrdset->red, rc->red); st->red = rc->red; } - rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); - rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); + rc->local = rrdvar_create_and_index("local", st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value); + rc->family = rrdvar_create_and_index("family", st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value); char fullname[RRDVAR_MAX_LENGTH + 1]; - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name); - rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); + snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_id(st), rrdcalc_name(rc)); + STRING *fullname_string = string_strdupz(fullname); + rc->hostid = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value); - snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name); - rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); + snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", rrdset_name(st), rrdcalc_name(rc)); + rc->hostname = rrdvar_create_and_index("host", host->rrdvar_root_index, fullname_string, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value); + + string_freez(fullname_string); if(rc->hostid && !rc->hostname) rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR; - if(!rc->units) rc->units = strdupz(st->units); + if(!rc->units) rc->units = string_dup(st->units); if (rc->original_info) { if (rc->info) - freez(rc->info); - rc->info = rrdcalc_replace_variables(rc->original_info, rc); + string_freez(rc->info); + + rc->info = rrdcalc_replace_variables(rrdcalc_original_info(rc), rc); } time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( host, rc->id, @@ -170,19 +175,20 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { rc->units, rc->info, 0, - 0); + rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0); + health_alarm_log(host, ae); } -static int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { - if((rc->hash_chart != st->hash || strcmp(rc->chart, st->id) != 0) && - (rc->hash_chart != st->hash_name || strcmp(rc->chart, st->name) != 0)) +static inline int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) { + if ( (rc->chart != st->id) + && (rc->chart != st->name)) return 0; - if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, st->module_name)) + if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, rrdset_module_name(st))) return 0; - if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name)) + if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, rrdset_plugin_name(st))) return 0; if (st->rrdhost->host_labels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->host_labels, rc->host_labels_pattern, '=')) @@ -197,7 +203,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) { // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id); RRDCALC *rc; - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost(host, rc) { if(unlikely(rc->rrdset)) continue; @@ -211,14 +217,15 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { RRDSET *st = rc->rrdset; if(!st) { - debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name); - error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name); + debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc)); + error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rrdcalc_chart_name(rc), rrdcalc_name(rc)); return; } RRDHOST *host = st->rrdhost; time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( host, rc->id, @@ -244,32 +251,24 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { rc->info, 0, 0); + health_alarm_log(host, ae); - debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname); + debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rrdcalc_chart_name(rc), rrdcalc_name(rc), rrdset_id(st), rrdhost_hostname(host)); // unlink it - if(rc->rrdset_prev) - rc->rrdset_prev->rrdset_next = rc->rrdset_next; - - if(rc->rrdset_next) - rc->rrdset_next->rrdset_prev = rc->rrdset_prev; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(st->alarms, rc, rrdset_prev, rrdset_next); - if(st->alarms == rc) - st->alarms = rc->rrdset_next; - - rc->rrdset_prev = rc->rrdset_next = NULL; - - rrdvar_free(host, &st->rrdvar_root_index, rc->local); + rrdvar_free(host, st->rrdvar_root_index, rc->local); rc->local = NULL; - rrdvar_free(host, &st->rrdfamily->rrdvar_root_index, rc->family); + rrdvar_free(host, st->rrdfamily->rrdvar_root_index, rc->family); rc->family = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rc->hostid); + rrdvar_free(host, host->rrdvar_root_index, rc->hostid); rc->hostid = NULL; - rrdvar_free(host, &host->rrdvar_root_index, rc->hostname); + rrdvar_free(host, host->rrdvar_root_index, rc->hostname); rc->hostname = NULL; rc->rrdset = NULL; @@ -280,18 +279,21 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { } RRDCALC *rrdcalc_find(RRDSET *st, const char *name) { - RRDCALC *rc; - uint32_t hash = simple_hash(name); + RRDCALC *rc = NULL; - for( rc = st->alarms; rc ; rc = rc->rrdset_next ) { - if(unlikely(rc->hash == hash && !strcmp(rc->name, name))) - return rc; + STRING *name_string = string_strdupz(name); + + foreach_rrdcalc_in_rrdset(st, rc) { + if(unlikely(rc->name == name_string)) + break; } - return NULL; + string_freez(name_string); + + return rc; } -inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) { +inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name) { RRDCALC *rc; if(unlikely(!chart)) { @@ -299,36 +301,39 @@ inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, ui return 1; } - if(unlikely(!hash_chart)) hash_chart = simple_hash(chart); - if(unlikely(!hash_name)) hash_name = simple_hash(name); + STRING *name_string = string_strdupz(name); + STRING *chart_string = string_strdupz(chart); + int ret = 0; // make sure it does not already exist - for(rc = host->alarms; rc ; rc = rc->next) { - if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) { - debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); - info("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); - return 1; + foreach_rrdcalc_in_rrdhost(host, rc) { + if (unlikely(rc->chart == chart_string && rc->name == name_string)) { + debug(D_HEALTH, "Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); + info("Health alarm '%s/%s' already exists in host '%s'.", chart, name, rrdhost_hostname(host)); + ret = 1; + break; } } - return 0; + string_freez(name_string); + string_freez(chart_string); + + return ret; } -inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) { - if(chart && name) { - uint32_t hash_chart = simple_hash(chart); - uint32_t hash_name = simple_hash(name); - - // re-use old IDs, by looking them up in the alarm log - ALARM_ENTRY *ae; - for(ae = host->health_log.alarms; ae ;ae = ae->next) { - if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) { - if(next_event_id) *next_event_id = ae->alarm_event_id + 1; - return ae->alarm_id; - } +inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { + // re-use old IDs, by looking them up in the alarm log + ALARM_ENTRY *ae = NULL; + for(ae = host->health_log.alarms; ae ;ae = ae->next) { + if(unlikely(name == ae->name && chart == ae->chart)) { + if(next_event_id) *next_event_id = ae->alarm_event_id + 1; + break; } } + if(ae) + return ae->alarm_id; + if (unlikely(!host->health_log.next_alarm_id)) host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); @@ -347,7 +352,7 @@ inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const ch * * @return It returns the new name on success and the old otherwise */ -char *alarm_name_with_dim(char *name, size_t namelen, const char *dim, size_t dimlen) { +char *alarm_name_with_dim(const char *name, size_t namelen, const char *dim, size_t dimlen) { char *newname,*move; newname = mallocz(namelen + dimlen + 2); @@ -407,15 +412,7 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { if(!rc->foreachdim) { // link it to the host alarms list - if(likely(host->alarms)) { - // append it - RRDCALC *t; - for(t = host->alarms; t && t->next ; t = t->next) ; - t->next = rc; - } - else { - host->alarms = rc; - } + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->host_alarms, rc, prev, next); // link it to its chart RRDSET *st; @@ -425,43 +422,33 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) { break; } } - } else { - //link it case there is a foreach - if(likely(host->alarms_with_foreach)) { - // append it - RRDCALC *t; - for(t = host->alarms_with_foreach; t && t->next ; t = t->next) ; - t->next = rc; - } - else { - host->alarms_with_foreach = rc; - } - - //I am not linking this alarm direct to the host here, this will be done when the children is created + } + else { + DOUBLE_LINKED_LIST_APPEND_UNSAFE(host->alarms_with_foreach, rc, prev, next); + // We are not linking this alarm directly to the host here + // It will eventually be done if it matches the dimensions } } inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) { - debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name); + debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rrdcalctemplate_name(rt)); - if(rrdcalc_exists(host, chart, rt->name, 0, 0)) + if(rrdcalc_exists(host, chart, rrdcalctemplate_name(rt))) return NULL; RRDCALC *rc = callocz(1, sizeof(RRDCALC)); rc->next_event_id = 1; - rc->name = strdupz(rt->name); - rc->hash = simple_hash(rc->name); - rc->chart = strdupz(chart); - rc->hash_chart = simple_hash(rc->chart); + rc->name = string_dup(rt->name); + rc->chart = string_strdupz(chart); uuid_copy(rc->config_hash_id, rt->config_hash_id); rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id); - if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions); - if(rt->foreachdim) { - rc->foreachdim = strdupz(rt->foreachdim); - rc->spdim = health_pattern_from_foreach(rc->foreachdim); - } + rc->dimensions = string_dup(rt->dimensions); + rc->foreachdim = string_dup(rt->foreachdim); + if(rt->foreachdim) + rc->spdim = health_pattern_from_foreach(rrdcalc_foreachdim(rc)); + rc->foreachcounter = rt->foreachcounter; rc->green = rt->green; @@ -485,55 +472,53 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, rc->update_every = rt->update_every; rc->options = rt->options; - if(rt->exec) rc->exec = strdupz(rt->exec); - if(rt->recipient) rc->recipient = strdupz(rt->recipient); - if(rt->source) rc->source = strdupz(rt->source); - if(rt->units) rc->units = strdupz(rt->units); - if(rt->info) { - rc->info = strdupz(rt->info); - rc->original_info = strdupz(rt->info); - } + rc->exec = stri