diff options
Diffstat (limited to 'database/rrdcontext.c')
| -rw-r--r-- | database/rrdcontext.c | 4249 |
1 files changed, 0 insertions, 4249 deletions
diff --git a/database/rrdcontext.c b/database/rrdcontext.c deleted file mode 100644 index 94d0d980ca..0000000000 --- a/database/rrdcontext.c +++ /dev/null @@ -1,4249 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "rrdcontext.h" -#include "sqlite/sqlite_context.h" -#include "aclk/schema-wrappers/context.h" -#include "aclk/aclk_contexts_api.h" -#include "aclk/aclk.h" -#include "storage_engine.h" - -#define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000 -#define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120 -#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS) -#define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10 - -#define LOG_TRANSITIONS false - -#define WORKER_JOB_HOSTS 1 -#define WORKER_JOB_CHECK 2 -#define WORKER_JOB_SEND 3 -#define WORKER_JOB_DEQUEUE 4 -#define WORKER_JOB_RETENTION 5 -#define WORKER_JOB_QUEUED 6 -#define WORKER_JOB_CLEANUP 7 -#define WORKER_JOB_CLEANUP_DELETE 8 -#define WORKER_JOB_PP_METRIC 9 // post-processing metrics -#define WORKER_JOB_PP_INSTANCE 10 // post-processing instances -#define WORKER_JOB_PP_CONTEXT 11 // post-processing contexts -#define WORKER_JOB_HUB_QUEUE_SIZE 12 -#define WORKER_JOB_PP_QUEUE_SIZE 13 - - -typedef enum __attribute__ ((__packed__)) { - RRD_FLAG_NONE = 0, - RRD_FLAG_DELETED = (1 << 0), // this is a deleted object (metrics, instances, contexts) - RRD_FLAG_COLLECTED = (1 << 1), // this object is currently being collected - RRD_FLAG_UPDATED = (1 << 2), // this object has updates to propagate - RRD_FLAG_ARCHIVED = (1 << 3), // this object is not currently being collected - RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET - RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database - RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub - RRD_FLAG_QUEUED_FOR_PP = (1 << 7), // this context is currently queued to be post-processed - RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API - - RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object - RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL - RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created - RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object - RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 13), // an instance or a metric switched RRDSET or RRDDIM - RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 14), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags - RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 15), // this object has no retention - RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 16), // this object changed its oldest time in the db - RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 17), // this object change its latest time in the db - RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 18), // this object has stopped being collected - RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 19), // this object has started being collected - RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 20), // this context belongs to a host that just disconnected - RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore - RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation - - // action to perform on an object - RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION = (1 << 30), // this object has to update its retention from the db -} RRD_FLAGS; - -#define RRD_FLAG_ALL_UPDATE_REASONS ( \ - RRD_FLAG_UPDATE_REASON_TRIGGERED \ - |RRD_FLAG_UPDATE_REASON_LOAD_SQL \ - |RRD_FLAG_UPDATE_REASON_NEW_OBJECT \ - |RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \ - |RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \ - |RRD_FLAG_UPDATE_REASON_CHANGED_METADATA \ - |RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \ - |RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T \ - |RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T \ - |RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \ - |RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \ - |RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \ - |RRD_FLAG_UPDATE_REASON_DB_ROTATION \ - |RRD_FLAG_UPDATE_REASON_UNUSED \ - ) - -#define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \ - RRD_FLAG_ARCHIVED \ - |RRD_FLAG_HIDDEN \ - |RRD_FLAG_ALL_UPDATE_REASONS \ - ) - -#define RRD_FLAGS_REQUIRED_FOR_DELETIONS ( \ - RRD_FLAG_DELETED \ - |RRD_FLAG_LIVE_RETENTION \ -) - -#define RRD_FLAGS_PREVENTING_DELETIONS ( \ - RRD_FLAG_QUEUED_FOR_HUB \ - |RRD_FLAG_COLLECTED \ - |RRD_FLAG_QUEUED_FOR_PP \ -) - -// get all the flags of an object -#define rrd_flags_get(obj) __atomic_load_n(&((obj)->flags), __ATOMIC_SEQ_CST) - -// check if ANY of the given flags (bits) is set -#define rrd_flag_check(obj, flag) (rrd_flags_get(obj) & (flag)) - -// check if ALL the given flags (bits) are set -#define rrd_flag_check_all(obj, flag) (rrd_flag_check(obj, flag) == (flag)) - -// set one or more flags (bits) -#define rrd_flag_set(obj, flag) __atomic_or_fetch(&((obj)->flags), flag, __ATOMIC_SEQ_CST) - -// clear one or more flags (bits) -#define rrd_flag_clear(obj, flag) __atomic_and_fetch(&((obj)->flags), ~(flag), __ATOMIC_SEQ_CST) - -// replace the flags of an object, with the supplied ones -#define rrd_flags_replace(obj, all_flags) __atomic_store_n(&((obj)->flags), all_flags, __ATOMIC_SEQ_CST) - -static inline void -rrd_flag_add_remove_atomic(RRD_FLAGS *flags, RRD_FLAGS check, RRD_FLAGS conditionally_add, RRD_FLAGS always_remove) { - RRD_FLAGS expected, desired; - - do { - expected = *flags; - - desired = expected; - desired &= ~(always_remove); - - if(!(expected & check)) - desired |= (check | conditionally_add); - - } while(!__atomic_compare_exchange_n(flags, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); -} - -#define rrd_flag_set_collected(obj) \ - rrd_flag_add_remove_atomic(&((obj)->flags) \ - /* check this flag */ \ - , RRD_FLAG_COLLECTED \ - \ - /* add these flags together with the above, if the above is not already set */ \ - , RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED | RRD_FLAG_UPDATED \ - \ - /* always remove these flags */ \ - , RRD_FLAG_ARCHIVED \ - | RRD_FLAG_DELETED \ - | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \ - | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \ - | RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \ - ) - -#define rrd_flag_set_archived(obj) \ - rrd_flag_add_remove_atomic(&((obj)->flags) \ - /* check this flag */ \ - , RRD_FLAG_ARCHIVED \ - \ - /* add these flags together with the above, if the above is not already set */ \ - , RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED | RRD_FLAG_UPDATED \ - \ - /* always remove these flags */ \ - , RRD_FLAG_COLLECTED \ - | RRD_FLAG_DELETED \ - | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \ - | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \ - ) - -#define rrd_flag_set_deleted(obj, reason) \ - rrd_flag_add_remove_atomic(&((obj)->flags) \ - /* check this flag */ \ - , RRD_FLAG_DELETED \ - \ - /* add these flags together with the above, if the above is not already set */ \ - , RRD_FLAG_UPDATE_REASON_ZERO_RETENTION | RRD_FLAG_UPDATED | (reason) \ - \ - /* always remove these flags */ \ - , RRD_FLAG_ARCHIVED \ - | RRD_FLAG_COLLECTED \ - ) - -#define rrd_flag_is_collected(obj) rrd_flag_check(obj, RRD_FLAG_COLLECTED) -#define rrd_flag_is_archived(obj) rrd_flag_check(obj, RRD_FLAG_ARCHIVED) -#define rrd_flag_is_deleted(obj) rrd_flag_check(obj, RRD_FLAG_DELETED) -#define rrd_flag_is_updated(obj) rrd_flag_check(obj, RRD_FLAG_UPDATED) - -// mark an object as updated, providing reasons (additional bits) -#define rrd_flag_set_updated(obj, reason) rrd_flag_set(obj, RRD_FLAG_UPDATED | (reason)) - -// clear an object as being updated, clearing also all the reasons -#define rrd_flag_unset_updated(obj) rrd_flag_clear(obj, RRD_FLAG_UPDATED | RRD_FLAG_ALL_UPDATE_REASONS) - - -static struct rrdcontext_reason { - RRD_FLAGS flag; - const char *name; - usec_t delay_ut; -} rrdcontext_reasons[] = { - // context related - {RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_CHANGED_METADATA, "changed metadata", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_ZERO_RETENTION, "has no retention", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T, "updated first_time_t", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T, "updated last_time_t", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED, "stopped collected", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED, "started collected", 5 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_UNUSED, "unused", 5 * USEC_PER_SEC }, - - // not context related - {RRD_FLAG_UPDATE_REASON_CHANGED_LINKING, "changed rrd link", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC }, - {RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION, "updated retention", 65 * USEC_PER_SEC }, - - // terminator - {0, NULL, 0 }, -}; - - -typedef struct rrdmetric { - uuid_t uuid; - - STRING *id; - STRING *name; - - RRDDIM *rrddim; - - time_t first_time_s; - time_t last_time_s; - RRD_FLAGS flags; - - struct rrdinstance *ri; -} RRDMETRIC; - -typedef struct rrdinstance { - uuid_t uuid; - - STRING *id; - STRING *name; - STRING *title; - STRING *units; - STRING *family; - uint32_t priority; - RRDSET_TYPE chart_type; - - RRD_FLAGS flags; // flags related to this instance - time_t first_time_s; - time_t last_time_s; - - time_t update_every_s; // data collection frequency - RRDSET *rrdset; // pointer to RRDSET when collected, or NULL - - DICTIONARY *rrdlabels; // linked to RRDSET->chart_labels or own version - - struct rrdcontext *rc; - DICTIONARY *rrdmetrics; - - struct { - uint32_t collected_metrics_count; // a temporary variable to detect BEGIN/END without SET - // don't use it for other purposes - // it goes up and then resets to zero, on every iteration - } internal; -} RRDINSTANCE; - -typedef struct rrdcontext { - uint64_t version; - - STRING *id; - STRING *title; - STRING *units; - STRING *family; - uint32_t priority; - RRDSET_TYPE chart_type; - - RRD_FLAGS flags; - time_t first_time_s; - time_t last_time_s; - - VERSIONED_CONTEXT_DATA hub; - - DICTIONARY *rrdinstances; - RRDHOST *rrdhost; - - struct { - RRD_FLAGS queued_flags; // the last flags that triggered the post-processing - usec_t queued_ut; // the last time this was queued - usec_t dequeued_ut; // the last time we sent (or deduplicated) this context - size_t executions; // how many times this context has been processed - } pp; - - struct { - RRD_FLAGS queued_flags; // the last flags that triggered the queueing - usec_t queued_ut; // the last time this was queued - usec_t delay_calc_ut; // the last time we calculated the scheduled_dispatched_ut - usec_t scheduled_dispatch_ut; // the time it was/is scheduled to be sent - usec_t dequeued_ut; // the last time we sent (or deduplicated) this context - size_t dispatches; // the number of times this has been dispatched to hub - } queue; - - netdata_mutex_t mutex; -} RRDCONTEXT; - -// ---------------------------------------------------------------------------- -// helper one-liners for RRDMETRIC - -static bool rrdmetric_update_retention(RRDMETRIC *rm); - -static inline RRDMETRIC *rrdmetric_acquired_value(RRDMETRIC_ACQUIRED *rma) { - return dictionary_acquired_item_value((DICTIONARY_ITEM *)rma); -} - -static inline RRDMETRIC_ACQUIRED *rrdmetric_acquired_dup(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return (RRDMETRIC_ACQUIRED *)dictionary_acquired_item_dup(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma); -} - -static inline void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - dictionary_acquired_item_release(rm->ri->rrdmetrics, (DICTIONARY_ITEM *)rma); -} - -const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return string2str(rm->id); -} - -const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return string2str(rm->name); -} - -STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return string_dup(rm->id); -} - -STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return string_dup(rm->name); -} - -NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - - if(rm->rrddim) - return rm->rrddim->last_stored_value; - - return NAN; -} - -// ---------------------------------------------------------------------------- -// helper one-liners for RRDINSTANCE - -static inline RRDINSTANCE *rrdinstance_acquired_value(RRDINSTANCE_ACQUIRED *ria) { - return dictionary_acquired_item_value((DICTIONARY_ITEM *)ria); -} - -static inline RRDINSTANCE_ACQUIRED *rrdinstance_acquired_dup(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return (RRDINSTANCE_ACQUIRED *)dictionary_acquired_item_dup(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria); -} - -static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - dictionary_acquired_item_release(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria); -} - -const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return string2str(ri->id); -} - -const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return string2str(ri->name); -} - -const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return string2str(ri->units); -} - -STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return string_dup(ri->units); -} - -DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return ri->rrdlabels; -} - -DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - if(!ri->rrdset) return NULL; - return ri->rrdset->functions_view; -} - -RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return ri->rc->rrdhost; -} - -// ---------------------------------------------------------------------------- -// helper one-liners for RRDCONTEXT - -static inline RRDCONTEXT *rrdcontext_acquired_value(RRDCONTEXT_ACQUIRED *rca) { - return dictionary_acquired_item_value((DICTIONARY_ITEM *)rca); -} - -const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca) { - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - return string2str(rc->id); -} - -bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host) { - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - return rc->rrdhost == host; -} - -bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - return ri->rc == rc; -} - -bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return rm->ri == ri; -} - -time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria) { - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - return ri->update_every_s; -} -time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - return rm->first_time_s; -} -time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma) { - RRDMETRIC *rm = rrdmetric_acquired_value(rma); - - if(rrd_flag_check(rm, RRD_FLAG_COLLECTED)) - return 0; - - return rm->last_time_s; -} - -static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquired_dup(RRDCONTEXT_ACQUIRED *rca) { - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca); -} - -static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) { - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - dictionary_acquired_item_release((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca); -} - -static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs); -static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs); - -#define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL) -static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*callback)(RRDCONTEXT *, bool, void *), bool snapshot, void *bundle); - -static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs); -static void rrdcontext_garbage_collect_for_all_hosts(void); - -#define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex)) -#define rrdcontext_unlock(rc) netdata_mutex_unlock(&((rc)->mutex)) - -// ---------------------------------------------------------------------------- -// Forward definitions - -static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc); -static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused); -static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused); - -static void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc); - -static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc); -static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function, RRD_FLAGS flags); -static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs); - -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function); -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function); -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function); - -// ---------------------------------------------------------------------------- -// visualizing flags - -static void rrd_flags_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) { - if(flags & RRD_FLAG_QUEUED_FOR_HUB) - buffer_json_add_array_item_string(wb, "QUEUED"); - - if(flags & RRD_FLAG_DELETED) - buffer_json_add_array_item_string(wb, "DELETED"); - - if(flags & RRD_FLAG_COLLECTED) - buffer_json_add_array_item_string(wb, "COLLECTED"); - - if(flags & RRD_FLAG_UPDATED) - buffer_json_add_array_item_string(wb, "UPDATED"); - - if(flags & RRD_FLAG_ARCHIVED) - buffer_json_add_array_item_string(wb, "ARCHIVED"); - - if(flags & RRD_FLAG_OWN_LABELS) - buffer_json_add_array_item_string(wb, "OWN_LABELS"); - - if(flags & RRD_FLAG_LIVE_RETENTION) - buffer_json_add_array_item_string(wb, "LIVE_RETENTION"); - - if(flags & RRD_FLAG_HIDDEN) - buffer_json_add_array_item_string(wb, "HIDDEN"); - - if(flags & RRD_FLAG_QUEUED_FOR_PP) - buffer_json_add_array_item_string(wb, "PENDING_UPDATES"); -} - -static void rrd_reasons_to_buffer_json_array_items(RRD_FLAGS flags, BUFFER *wb) { - for(int i = 0, added = 0; rrdcontext_reasons[i].name ; i++) { - if (flags & rrdcontext_reasons[i].flag) { - buffer_json_add_array_item_string(wb, rrdcontext_reasons[i].name); - added++; - } - } -} - -// ---------------------------------------------------------------------------- -// RRDMETRIC - -// free the contents of RRDMETRIC. -// RRDMETRIC itself is managed by DICTIONARY - no need to free it here. -static void rrdmetric_free(RRDMETRIC *rm) { - string_freez(rm->id); - string_freez(rm->name); - - rm->id = NULL; - rm->name = NULL; - rm->ri = NULL; -} - -// called when this rrdmetric is inserted to the rrdmetrics dictionary of a rrdinstance -// the constructor of the rrdmetric object -static void rrdmetric_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance) { - RRDMETRIC *rm = value; - - // link it to its parent - rm->ri = rrdinstance; - - // remove flags that we need to figure out at runtime - rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics - - // signal the react callback to do the job - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_NEW_OBJECT); -} - -// called when this rrdmetric is deleted from the rrdmetrics dictionary of a rrdinstance -// the destructor of the rrdmetric object -static void rrdmetric_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) { - RRDMETRIC *rm = value; - - internal_error(rm->rrddim, "RRDMETRIC: '%s' is freed but there is a RRDDIM linked to it.", string2str(rm->id)); - - // free the resources - rrdmetric_free(rm); -} - -// called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance -// while this is called, the dictionary is write locked, but there may be other users of the object -static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdinstance __maybe_unused) { - RRDMETRIC *rm = old_value; - RRDMETRIC *rm_new = new_value; - - internal_error(rm->id != rm_new->id, - "RRDMETRIC: '%s' cannot change id to '%s'", - string2str(rm->id), string2str(rm_new->id)); - - if(uuid_compare(rm->uuid, rm_new->uuid) != 0) { -#ifdef NETDATA_INTERNAL_CHECKS - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; - uuid_unparse(rm->uuid, uuid1); - uuid_unparse(rm_new->uuid, uuid2); - - time_t old_first_time_s = 0; - time_t old_last_time_s = 0; - if(rrdmetric_update_retention(rm)) { - old_first_time_s = rm->first_time_s; - old_last_time_s = rm->last_time_s; - } - - uuid_copy(rm->uuid, rm_new->uuid); - - time_t new_first_time_s = 0; - time_t new_last_time_s = 0; - if(rrdmetric_update_retention(rm)) { - new_first_time_s = rm->first_time_s; - new_last_time_s = rm->last_time_s; - } - - internal_error(true, - "RRDMETRIC: '%s' of instance '%s' of host '%s' changed UUID from '%s' (retention %ld to %ld, %ld secs) to '%s' (retention %ld to %ld, %ld secs)" - , string2str(rm->id) - , string2str(rm->ri->id) - , rrdhost_hostname(rm->ri->rc->rrdhost) - , uuid1, old_first_time_s, old_last_time_s, old_last_time_s - old_first_time_s - , uuid2, new_first_time_s, new_last_time_s, new_last_time_s - new_first_time_s - ); -#else - uuid_copy(rm->uuid, rm_new->uuid); -#endif - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(rm->rrddim && rm_new->rrddim && rm->rrddim != rm_new->rrddim) { - rm->rrddim = rm_new->rrddim; - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING); - } - -#ifdef NETDATA_INTERNAL_CHECKS - if(rm->rrddim && uuid_compare(rm->uuid, rm->rrddim->metric_uuid) != 0) { - char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN]; - uuid_unparse(rm->uuid, uuid1); - uuid_unparse(rm_new->uuid, uuid2); - internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2); - } -#endif - - if(rm->rrddim != rm_new->rrddim) - rm->rrddim = rm_new->rrddim; - - if(rm->name != rm_new->name) { - STRING *old = rm->name; - rm->name = string_dup(rm_new->name); - string_freez(old); - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(!rm->first_time_s || (rm_new->first_time_s && rm_new->first_time_s < rm->first_time_s)) { - rm->first_time_s = rm_new->first_time_s; - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T); - } - - if(!rm->last_time_s || (rm_new->last_time_s && rm_new->last_time_s > rm->last_time_s)) { - rm->last_time_s = rm_new->last_time_s; - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T); - } - - rrd_flag_set(rm, rm_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no needs for atomics on rm_new - - if(rrd_flag_is_collected(rm) && rrd_flag_is_archived(rm)) - rrd_flag_set_collected(rm); - - if(rrd_flag_check(rm, RRD_FLAG_UPDATED)) - rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT); - - rrdmetric_free(rm_new); - - // the react callback will continue from here - return rrd_flag_is_updated(rm); -} - -// this is called after the insert or the conflict callbacks, -// but the dictionary is now unlocked -static void rrdmetric_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdinstance __maybe_unused) { - RRDMETRIC *rm = value; - rrdmetric_trigger_updates(rm, __FUNCTION__ ); -} - -static void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) { - if(unlikely(!ri)) return; - if(likely(ri->rrdmetrics)) return; - - ri->rrdmetrics = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - &dictionary_stats_category_rrdcontext, sizeof(RRDMETRIC)); - - dictionary_register_insert_callback(ri->rrdmetrics, rrdmetric_insert_callback, ri); - dictionary_register_delete_callback(ri->rrdmetrics, rrdmetric_delete_callback, ri); - dictionary_register_conflict_callback(ri->rrdmetrics, rrdmetric_conflict_callback, ri); - dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, ri); -} - -static void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) { - if(unlikely(!ri || !ri->rrdmetrics)) return; - dictionary_destroy(ri->rrdmetrics); - ri->rrdmetrics = NULL; -} - -// trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) { - if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD))) - rrd_flag_set_archived(rm); - - if(rrd_flag_is_updated(rm) || !rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)) { - rrd_flag_set_updated(rm->ri, RRD_FLAG_UPDATE_REASON_TRIGGERED); - rrdcontext_queue_for_post_processing(rm->ri->rc, function, rm->flags); - } -} - -// ---------------------------------------------------------------------------- -// RRDMETRIC HOOKS ON RRDDIM - -static inline void rrdmetric_from_rrddim(RRDDIM *rd) { - if(unlikely(!rd->rrdset)) - fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd)); - - if(unlikely(!rd->rrdset->rrdhost)) - fatal("RRDMETRIC: rrdset '%s' does not have a rrdhost", rrdset_id(rd->rrdset)); - - if(unlikely(!rd->rrdset->rrdinstance)) - fatal("RRDMETRIC: rrdset '%s' does not have a rrdinstance", rrdset_id(rd->rrdset)); - - RRDINSTANCE *ri = rrdinstance_acquired_value(rd->rrdset->rrdinstance); - - RRDMETRIC trm = { - .id = string_dup(rd->id), - .name = string_dup(rd->name), - .flags = RRD_FLAG_NONE, // no need for atomics - .rrddim = rd, - }; - uuid_copy(trm.uuid, rd->metric_uuid); - - RRDMETRIC_ACQUIRED *rma = (RRDMETRIC_ACQUIRED *)dictionary_set_and_acquire_item(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm)); - - if(rd->rrdmetric) - rrdmetric_release(rd->rrdmetric); - - rd->rrdmetric = rma; -} - -#define rrddim_get_rrdmetric(rd) rrddim_get_rrdmetric_with_trace(rd, __FUNCTION__) -static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char *function) { - if(unlikely(!rd->rrdmetric)) { - error("RRDMETRIC: RRDDIM '%s' is not linked to an RRDMETRIC at %s()", rrddim_id(rd), function); - return NULL; - } - - RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric); - if(unlikely(!rm)) { - error("RRDMETRIC: RRDDIM '%s' lost the link to its RRDMETRIC at %s()", rrddim_id(rd), function); - return NULL; - } - - if(unlikely(rm->rrddim != rd)) - fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function); - - return rm; -} - -static inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) { - RRDMETRIC *rm = rrddim_get_rrdmetric(rd); - if(unlikely(!rm)) return; - - if(unlikely(rrd_flag_is_collected(rm))) - rrd_flag_set_archived(rm); - - rm->rrddim = NULL; - rrdmetric_trigger_updates(rm, __FUNCTION__ ); - rrdmetric_release(rd->rrdmetric); - rd->rrdmetric = NULL; -} - -static inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) { - RRDMETRIC *rm = rrddim_get_rrdmetric(rd); - if(unlikely(!rm)) return; - - if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED|RRDDIM_FLAG_OBSOLETE))) { - if(unlikely(rrd_flag_is_collected(rm))) - rrd_flag_set_archived(rm); - } - - rrdmetric_trigger_updates(rm, __FUNCTION__ ); -} - -static inline void rrdmetric_collected_rrddim(RRDDIM *rd) { - RRDMETRIC *rm = rrddim_get_rrdmetric(rd); - if(unlikely(!rm)) return; - - if(unlikely(!rrd_flag_is_collected(rm))) - rrd_flag_set_collected(rm); - - // we use this variable to detect BEGIN/END without SET - rm->ri->internal.collected_metrics_count++; - - rrdmetric_trigger_updates(rm, __FUNCTION__ ); -} - -// ---------------------------------------------------------------------------- -// RRDINSTANCE - -static void rrdinstance_free(RRDINSTANCE *ri) { - - if(rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) - dictionary_destroy(ri->rrdlabels); - - rrdmetrics_destroy_from_rrdinstance(ri); - string_freez(ri->id); - string_freez(ri->name); - string_freez(ri->title); - string_freez(ri->units); - string_freez(ri->family); - - ri->id = NULL; - ri->name = NULL; - ri->title = NULL; - ri->units = NULL; - ri->family = NULL; - ri->rc = NULL; - ri->rrdlabels = NULL; - ri->rrdmetrics = NULL; - ri->rrdset = NULL; -} - -static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *rrdcontext) { - RRDINSTANCE *ri = value; - - // link it to its parent - ri->rc = rrdcontext; - - ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics - - if(!ri->name) - ri->name = string_dup(ri->id); - - if(ri->rrdset) { - ri->rrdlabels = ri->rrdset->rrdlabels; - ri->flags &= ~RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor - } - else { - ri->rrdlabels = rrdlabels_create(); - ri->flags |= RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor - } - - if(ri->rrdset) { - if(unlikely(rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN))) - ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor - else - ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor - } - - rrdmetrics_create_in_rrdinstance(ri); - - // signal the react callback to do the job - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT); -} - -static void rrdinstance_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *va |