diff options
Diffstat (limited to 'database/rrddim.c')
| -rw-r--r-- | database/rrddim.c | 443 |
1 files changed, 210 insertions, 233 deletions
diff --git a/database/rrddim.c b/database/rrddim.c index 0f99f98df7..57c18d07e8 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -1,8 +1,181 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#define NETDATA_RRD_INTERNALS #include "rrd.h" -#include "storage_engine.h" + +// ---------------------------------------------------------------------------- +// compatibility layer for RRDDIM files v019 + +#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019" + +struct avl_element_v019 { + void *avl_link[2]; + signed char avl_balance; +}; + +struct rrddim_map_save_v019 { + struct avl_element_v019 avl; // ignored + void *id; // ignored + void *name; // ignored + uint32_t algorithm; // print warning on mismatch - update on load + uint32_t storage_engine_id; // ignored + long long multiplier; // print warning on mismatch - update on load + long long divisor; // print warning on mismatch - update on load + uint32_t flags; // ignored + uint32_t hash; // ignored + uint32_t hash_name; // ignored + void *cache_filename; // ignored - we use it to keep the filename to save back + size_t collections_counter; // ignored + void *state; // ignored + size_t unused[8]; // ignored + long long collected_value_max; // ignored + unsigned int updated:1; // ignored + unsigned int exposed:1; // ignored + struct timeval last_collected_time; // check to reset all - ignored after load + long double calculated_value; // ignored + long double last_calculated_value; // ignored + long double last_stored_value; // ignored + long long collected_value; // ignored + long long last_collected_value; // load and save + long double collected_volume; // ignored + long double stored_volume; // ignored + void *next; // ignored + void *rrdset; // ignored + long entries; // check to reset all - update on load + int update_every; // check to reset all - update on load + size_t memsize; // check to reset all - update on load + char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load + void *variables; // ignored + storage_number values[]; // the array of values +}; + +size_t rrddim_memory_file_header_size(void) { + return sizeof(struct rrddim_map_save_v019); +} + +void rrddim_memory_file_update(RRDDIM *rd) { + if(!rd || !rd->db.rd_on_file) return; + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; + + rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec; + rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec; + rd_on_file->last_collected_value = rd->collector.last_collected_value; +} + +void rrddim_memory_file_free(RRDDIM *rd) { + if(!rd || !rd->db.rd_on_file) return; + + // needed for storage engine MAP, to save the latest state + rrddim_memory_file_update(rd); + + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; + __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); + freez(rd_on_file->cache_filename); + netdata_munmap(rd_on_file, rd_on_file->memsize); + + // remove the pointers from the RRDDIM + rd->db.rd_on_file = NULL; + rd->db.data = NULL; +} + +const char *rrddim_cache_filename(RRDDIM *rd) { + if(!rd || !rd->db.rd_on_file) return NULL; + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; + return rd_on_file->cache_filename; +} + +void rrddim_memory_file_save(RRDDIM *rd) { + if(!rd || !rd->db.rd_on_file) return; + + rrddim_memory_file_update(rd); + + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; + if(rd_on_file->storage_engine_id != STORAGE_ENGINE_SAVE) return; + + memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize); +} + +static bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd) { + assert(st->storage_engine_id == STORAGE_ENGINE_SAVE || st->storage_engine_id == STORAGE_ENGINE_MAP); + + struct rrddim_map_save_v019 *rd_on_file = NULL; + + unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number)); + + char filename[FILENAME_MAX + 1]; + char fullfilename[FILENAME_MAX + 1]; + rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX); + snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename); + + rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap( + fullfilename, size, ((st->storage_engine_id == STORAGE_ENGINE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL); + + if(unlikely(!rd_on_file)) return false; + + struct timeval now; + now_realtime_timeval(&now); + + int reset = 0; + rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0'; + if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) { + netdata_log_info("Initializing file %s.", fullfilename); + memset(rd_on_file, 0, size); + reset = 1; + } + else if(rd_on_file->memsize != size) { + netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize); + memset(rd_on_file, 0, size); + reset = 1; + } + else if(rd_on_file->update_every != st->update_every) { + netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every); + memset(rd_on_file, 0, size); + reset = 1; + } + else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) { + netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every); + memset(rd_on_file, 0, size); + reset = 1; + } + + if(!reset) { + rd->collector.last_collected_value = rd_on_file->last_collected_value; + + if(rd_on_file->algorithm != rd->algorithm) + netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.", + fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm)); + + if(rd_on_file->multiplier != rd->multiplier) + netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). " + "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier); + + if(rd_on_file->divisor != rd->divisor) + netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). " + "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor); + } + + // zero the entire header + memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019)); + + // set the important fields + strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019); + rd_on_file->algorithm = rd->algorithm; + rd_on_file->multiplier = rd->multiplier; + rd_on_file->divisor = rd->divisor; + rd_on_file->entries = st->db.entries; + rd_on_file->update_every = rd->rrdset->update_every; + rd_on_file->memsize = size; + rd_on_file->storage_engine_id = st->storage_engine_id; + rd_on_file->cache_filename = strdupz(fullfilename); + + __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); + + rd->db.data = &rd_on_file->values[0]; + rd->db.rd_on_file = rd_on_file; + rd->db.memsize = size; + rrddim_memory_file_update(rd); + + return true; +} // ---------------------------------------------------------------------------- // RRDDIM index @@ -14,7 +187,6 @@ struct rrddim_constructor { collected_number multiplier; collected_number divisor; RRD_ALGORITHM algorithm; - RRD_MEMORY_MODE memory_mode; enum { RRDDIM_REACT_NONE = 0, @@ -26,7 +198,7 @@ struct rrddim_constructor { // isolated call to appear // separate in statistics -static void *rrddim_alloc_db(size_t entries) { +static storage_number *rrddim_alloc_db(size_t entries) { return callocz(entries, sizeof(storage_number)); } @@ -51,21 +223,22 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) rd->collector.counter = 1; - if(ctr->memory_mode == RRD_MEMORY_MODE_MAP || ctr->memory_mode == RRD_MEMORY_MODE_SAVE) { - if(!rrddim_memory_load_or_create_map_save(st, rd, ctr->memory_mode)) { - netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (ctr->memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st), rrddim_name(rd)); - ctr->memory_mode = RRD_MEMORY_MODE_RAM; + if (st->storage_engine_id == STORAGE_ENGINE_MAP || st->storage_engine_id == STORAGE_ENGINE_SAVE) { + if(!rrddim_memory_load_or_create_map_save(st, rd)) { + netdata_log_info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", + storage_engine_name(st->storage_engine_id), rrdset_name(st), rrddim_name(rd)); + st->storage_engine_id = STORAGE_ENGINE_RAM; } } - if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) { + if (st->storage_engine_id == STORAGE_ENGINE_RAM) { size_t entries = st->db.entries; if(!entries) entries = 5; rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL); if(!rd->db.data) { netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); - ctr->memory_mode = RRD_MEMORY_MODE_ALLOC; + st->storage_engine_id = STORAGE_ENGINE_ALLOC; } else { rd->db.memsize = entries * sizeof(storage_number); @@ -73,7 +246,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v } } - if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) { + if (st->storage_engine_id == STORAGE_ENGINE_ALLOC || st->storage_engine_id == STORAGE_ENGINE_NONE) { size_t entries = st->db.entries; if(entries < 5) entries = 5; @@ -82,19 +255,14 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED); } - rd->rrd_memory_mode = ctr->memory_mode; - if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) uuid_generate(rd->metric_uuid); // initialize the db tiers { size_t initialized = 0; - for(size_t tier = 0; tier < storage_tiers ; tier++) { - STORAGE_ENGINE *eng = host->db[tier].eng; - rd->tiers[tier].backend = eng->backend; - rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping; - rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance); + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) { + rd->tiers[tier].db_metric_handle = storage_engine_metric_get_or_create(rd, host->db[tier].id, host->db[tier].instance); storage_point_unset(rd->tiers[tier].virtual_point); initialized++; @@ -111,10 +279,10 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v // initialize data collection for all tiers { size_t initialized = 0; - for (size_t tier = 0; tier < storage_tiers; tier++) { + for (size_t tier = 0; tier < rrdb.storage_tiers; tier++) { if (rd->tiers[tier].db_metric_handle) { rd->tiers[tier].db_collection_handle = - storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); + storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); initialized++; } } @@ -169,13 +337,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) { size_t tiers_available = 0, tiers_said_no_retention = 0; - for(size_t tier = 0; tier < storage_tiers ;tier++) { + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) { if(!rd->tiers[tier].db_collection_handle) continue; tiers_available++; - if(storage_engine_store_finalize(rd->tiers[tier].db_collection_handle)) + if(storage_engine_store_finalize(rd->rrdset->storage_engine_id, rd->tiers[tier].db_collection_handle)) tiers_said_no_retention++; rd->tiers[tier].db_collection_handle = NULL; @@ -199,7 +367,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v netdata_log_debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); - if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (!rrddim_finalize_collection_and_check_retention(rd) && st->storage_engine_id == STORAGE_ENGINE_DBENGINE) { /* This metric has no data and no references */ metaqueue_delete_dimension_uuid(&rd->metric_uuid); } @@ -215,18 +383,17 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures rrddim_memory_file_free(rd); - for(size_t tier = 0; tier < storage_tiers ;tier++) { + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) { if(!rd->tiers[tier].db_metric_handle) continue; - STORAGE_ENGINE* eng = host->db[tier].eng; - eng->api.metric_release(rd->tiers[tier].db_metric_handle); + storage_engine_metric_release(host->db[tier].id, rd->tiers[tier].db_metric_handle); rd->tiers[tier].db_metric_handle = NULL; } if(rd->db.data) { __atomic_sub_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED); - if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) + if(st->storage_engine_id == STORAGE_ENGINE_RAM) netdata_munmap(rd->db.data, rd->db.memsize); else freez(rd->db.data); @@ -250,10 +417,10 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, rc += rrddim_set_multiplier(st, rd, ctr->multiplier); rc += rrddim_set_divisor(st, rd, ctr->divisor); - for(size_t tier = 0; tier < storage_tiers ;tier++) { + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) { if (!rd->tiers[tier].db_collection_handle) rd->tiers[tier].db_collection_handle = - storage_metric_store_init(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); + storage_metric_store_init(st->storage_engine_id, rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); } if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { @@ -290,7 +457,7 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo } size_t rrddim_size(void) { - return sizeof(RRDDIM) + storage_tiers * sizeof(struct rrddim_tier); + return sizeof(RRDDIM) + rrdb.storage_tiers * sizeof(struct rrddim_tier); } void rrddim_index_init(RRDSET *st) { @@ -317,7 +484,7 @@ static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id) { // ---------------------------------------------------------------------------- // RRDDIM - find a dimension -inline RRDDIM *rrddim_find(RRDSET *st, const char *id) { +inline RRDDIM *rrddim_find_by_id(RRDSET *st, const char *id) { netdata_log_debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", rrdset_name(st), id); return rrddim_index_find(st, id); @@ -344,15 +511,6 @@ void rrddim_acquired_release(RRDDIM_ACQUIRED *rda) { dictionary_acquired_item_release(rd->rrdset->rrddim_root_index, (const DICTIONARY_ITEM *)rda); } -// This will not return dimensions that are archived -RRDDIM *rrddim_find_active(RRDSET *st, const char *id) { - RRDDIM *rd = rrddim_find(st, id); - - if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))) - return NULL; - - return rd; -} // ---------------------------------------------------------------------------- // RRDDIM rename a dimension @@ -419,17 +577,17 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor) { // ---------------------------------------------------------------------------- time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) { - if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle)) + if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle)) return 0; - return storage_engine_latest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle); + return storage_engine_latest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle); } // get the timestamp of the last entry in the round-robin database time_t rrddim_last_entry_s(RRDDIM *rd) { time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0); - for(size_t tier = 1; tier < storage_tiers ;tier++) { + for (size_t tier = 1; tier < rrdb.storage_tiers ;tier++) { if(unlikely(!rd->tiers[tier].db_metric_handle)) continue; time_t t = rrddim_last_entry_s_of_tier(rd, tier); @@ -441,16 +599,16 @@ time_t rrddim_last_entry_s(RRDDIM *rd) { } time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) { - if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle)) + if (unlikely(tier > rrdb.storage_tiers || !rd->tiers[tier].db_metric_handle)) return 0; - return storage_engine_oldest_time_s(rd->tiers[tier].backend, rd->tiers[tier].db_metric_handle); + return storage_engine_oldest_time_s(rd->rrdset->storage_engine_id, rd->tiers[tier].db_metric_handle); } time_t rrddim_first_entry_s(RRDDIM *rd) { time_t oldest_time_s = 0; - for(size_t tier = 0; tier < storage_tiers ;tier++) { + for (size_t tier = 0; tier < rrdb.storage_tiers ;tier++) { time_t t = rrddim_first_entry_s_of_tier(rd, tier); if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s)) oldest_time_s = t; @@ -459,14 +617,9 @@ time_t rrddim_first_entry_s(RRDDIM *rd) { return oldest_time_s; } -RRDDIM *rrddim_add_custom(RRDSET *st - , const char *id - , const char *name - , collected_number multiplier - , collected_number divisor - , RRD_ALGORITHM algorithm - , RRD_MEMORY_MODE memory_mode - ) { +RRDDIM *rrddim_add(RRDSET *st ,const char *id, const char *name, + collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm) +{ struct rrddim_constructor tmp = { .st = st, .id = id, @@ -474,7 +627,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st .multiplier = multiplier, .divisor = divisor, .algorithm = algorithm, - .memory_mode = memory_mode, }; RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, rrddim_size(), &tmp); @@ -497,7 +649,7 @@ int rrddim_hide(RRDSET *st, const char *id) { RRDHOST *host = st->rrdhost; - RRDDIM *rd = rrddim_find(st, id); + RRDDIM *rd = rrddim_find_by_id(st, id); if(unlikely(!rd)) { netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 1; @@ -516,7 +668,7 @@ int rrddim_unhide(RRDSET *st, const char *id) { netdata_log_debug(D_RRD_CALLS, "rrddim_unhide() for chart %s, dimension %s", rrdset_name(st), id); RRDHOST *host = st->rrdhost; - RRDDIM *rd = rrddim_find(st, id); + RRDDIM *rd = rrddim_find_by_id(st, id); if(unlikely(!rd)) { netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 1; @@ -580,7 +732,7 @@ collected_number rrddim_timed_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM * collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) { RRDHOST *host = st->rrdhost; - RRDDIM *rd = rrddim_find(st, id); + RRDDIM *rd = rrddim_find_by_id(st, id); if(unlikely(!rd)) { netdata_log_error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, rrdset_name(st), rrdset_id(st), rrdhost_hostname(host)); return 0; @@ -590,178 +742,3 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) } -// ---------------------------------------------------------------------------- -// compatibility layer for RRDDIM files v019 - -#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019" - -struct avl_element_v019 { - void *avl_link[2]; - signed char avl_balance; -}; - -struct rrddim_map_save_v019 { - struct avl_element_v019 avl; // ignored - void *id; // ignored - void *name; // ignored - uint32_t algorithm; // print warning on mismatch - update on load - uint32_t rrd_memory_mode; // ignored - long long multiplier; // print warning on mismatch - update on load - long long divisor; // print warning on mismatch - update on load - uint32_t flags; // ignored - uint32_t hash; // ignored - uint32_t hash_name; // ignored - void *cache_filename; // ignored - we use it to keep the filename to save back - size_t collections_counter; // ignored - void *state; // ignored - size_t unused[8]; // ignored - long long collected_value_max; // ignored - unsigned int updated:1; // ignored - unsigned int exposed:1; // ignored - struct timeval last_collected_time; // check to reset all - ignored after load - long double calculated_value; // ignored - long double last_calculated_value; // ignored - long double last_stored_value; // ignored - long long collected_value; // ignored - long long last_collected_value; // load and save - long double collected_volume; // ignored - long double stored_volume; // ignored - void *next; // ignored - void *rrdset; // ignored - long entries; // check to reset all - update on load - int update_every; // check to reset all - update on load - size_t memsize; // check to reset all - update on load - char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load - void *variables; // ignored - storage_number values[]; // the array of values -}; - -size_t rrddim_memory_file_header_size(void) { - return sizeof(struct rrddim_map_save_v019); -} - -void rrddim_memory_file_update(RRDDIM *rd) { - if(!rd || !rd->db.rd_on_file) return; - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; - - rd_on_file->last_collected_time.tv_sec = rd->collector.last_collected_time.tv_sec; - rd_on_file->last_collected_time.tv_usec = rd->collector.last_collected_time.tv_usec; - rd_on_file->last_collected_value = rd->collector.last_collected_value; -} - -void rrddim_memory_file_free(RRDDIM *rd) { - if(!rd || !rd->db.rd_on_file) return; - - // needed for memory mode map, to save the latest state - rrddim_memory_file_update(rd); - - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; - __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); - freez(rd_on_file->cache_filename); - netdata_munmap(rd_on_file, rd_on_file->memsize); - - // remove the pointers from the RRDDIM - rd->db.rd_on_file = NULL; - rd->db.data = NULL; -} - -const char *rrddim_cache_filename(RRDDIM *rd) { - if(!rd || !rd->db.rd_on_file) return NULL; - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; - return rd_on_file->cache_filename; -} - -void rrddim_memory_file_save(RRDDIM *rd) { - if(!rd || !rd->db.rd_on_file) return; - - rrddim_memory_file_update(rd); - - struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file; - if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return; - - memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize); -} - -bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) { - if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP) - return false; - - struct rrddim_map_save_v019 *rd_on_file = NULL; - - unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number)); - - char filename[FILENAME_MAX + 1]; - char fullfilename[FILENAME_MAX + 1]; - rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX); - snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename); - - rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap( - fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL); - - if(unlikely(!rd_on_file)) return false; - - struct timeval now; - now_realtime_timeval(&now); - - int reset = 0; - rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0'; - if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) { - netdata_log_info("Initializing file %s.", fullfilename); - memset(rd_on_file, 0, size); - reset = 1; - } - else if(rd_on_file->memsize != size) { - netdata_log_error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, (unsigned long int) rd_on_file->memsize); - memset(rd_on_file, 0, size); - reset = 1; - } - else if(rd_on_file->update_every != st->update_every) { - netdata_log_error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every); - memset(rd_on_file, 0, size); - reset = 1; - } - else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) { - netdata_log_info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every); - memset(rd_on_file, 0, size); - reset = 1; - } - - if(!reset) { - rd->collector.last_collected_value = rd_on_file->last_collected_value; - - if(rd_on_file->algorithm != rd->algorithm) - netdata_log_info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.", - fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm)); - - if(rd_on_file->multiplier != rd->multiplier) - netdata_log_info("File %s does not have the expected multiplier (expected %d, found %ld). " - "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier); - - if(rd_on_file->divisor != rd->divisor) - netdata_log_info("File %s does not have the expected divisor (expected %d, found %ld). " - "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor); - } - - // zero the entire header - memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019)); - - // set the important fields - strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019); - rd_on_file->algorithm = rd->algorithm; - rd_on_file->multiplier = rd->multiplier; - rd_on_file->divisor = rd->divisor; - rd_on_file->entries = st->db.entries; - rd_on_file->update_every = rd->rrdset->update_every; - rd_on_file->memsize = size; - rd_on_file->rrd_memory_mode = memory_mode; - rd_on_file->cache_filename = strdupz(fullfilename); - - __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); - - rd->db.data = &rd_on_file->values[0]; - rd->db.rd_on_file = rd_on_file; - rd->db.memsize = size; - rrddim_memory_file_update(rd); - - return true; -} |