summaryrefslogtreecommitdiffstats
path: root/database/rrdhost.c
diff options
context:
space:
mode:
Diffstat (limited to 'database/rrdhost.c')
-rw-r--r--database/rrdhost.c702
1 files changed, 132 insertions, 570 deletions
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 235d64b1ee..32226629ac 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -1,37 +1,24 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#define NETDATA_RRD_INTERNALS
#include "rrd.h"
static void rrdhost_streaming_sender_structures_init(RRDHOST *host);
-bool dbengine_enabled = false; // will become true if and when dbengine is initialized
-size_t storage_tiers = 3;
-bool use_direct_io = true;
-size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
-RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
-
#if RRD_STORAGE_TIERS != 5
#error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier
#endif
size_t get_tier_grouping(size_t tier) {
- if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1;
+ if(unlikely(tier >= rrdb.storage_tiers)) tier = rrdb.storage_tiers - 1;
size_t grouping = 1;
// first tier is always 1 iteration of whatever update every the chart has
for(size_t i = 1; i <= tier ;i++)
- grouping *= storage_tiers_grouping_iterations[i];
+ grouping *= rrdb.storage_tiers_grouping_iterations[i];
return grouping;
}
-RRDHOST *localhost = NULL;
-netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;
-
-time_t rrdset_free_obsolete_time_s = 3600;
-time_t rrdhost_free_orphan_time_s = 3600;
-
bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
#ifdef ENABLE_DBENGINE
if(!rrdeng_is_legacy(engine))
@@ -41,14 +28,13 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
return false;
}
-RRDHOST *find_host_by_node_id(char *node_id) {
-
+RRDHOST *rrdhost_find_by_node_id(const char *node_id) {
uuid_t node_uuid;
if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
return NULL;
RRDHOST *host, *ret = NULL;
- dfe_start_read(rrdhost_root_index, host) {
+ dfe_start_read(rrdb.rrdhost_root_index, host) {
if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
ret = host;
break;
@@ -62,27 +48,10 @@ RRDHOST *find_host_by_node_id(char *node_id) {
// ----------------------------------------------------------------------------
// RRDHOST indexes management
-DICTIONARY *rrdhost_root_index = NULL;
-static DICTIONARY *rrdhost_root_index_hostname = NULL;
-
-static inline void rrdhost_init() {
- if(unlikely(!rrdhost_root_index)) {
- rrdhost_root_index = dictionary_create_advanced(
- DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
- &dictionary_stats_category_rrdhost, 0);
- }
-
- if(unlikely(!rrdhost_root_index_hostname)) {
- rrdhost_root_index_hostname = dictionary_create_advanced(
- DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE,
- &dictionary_stats_category_rrdhost, 0);
- }
-}
-
RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid) {
netdata_log_debug(D_RRD_CALLS, "rrdhost_find_and_acquire() host %s", machine_guid);
- return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
+ return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdb.rrdhost_root_index, machine_guid);
}
RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha) {
@@ -96,22 +65,22 @@ void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha) {
if(unlikely(!rha))
return;
- dictionary_acquired_item_release(rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
+ dictionary_acquired_item_release(rrdb.rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
}
// ----------------------------------------------------------------------------
// RRDHOST index by UUID
inline size_t rrdhost_hosts_available(void) {
- return dictionary_entries(rrdhost_root_index);
+ return dictionary_entries(rrdb.rrdhost_root_index);
}
inline RRDHOST *rrdhost_find_by_guid(const char *guid) {
- return dictionary_get(rrdhost_root_index, guid);
+ return dictionary_get(rrdb.rrdhost_root_index, guid);
}
static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
- RRDHOST *ret_machine_guid = dictionary_set(rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
+ RRDHOST *ret_machine_guid = dictionary_set(rrdb.rrdhost_root_index, host->machine_guid, host, sizeof(RRDHOST));
if(ret_machine_guid == host)
rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID);
else {
@@ -125,7 +94,7 @@ static inline RRDHOST *rrdhost_index_add_by_guid(RRDHOST *host) {
static void rrdhost_index_del_by_guid(RRDHOST *host) {
if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_MACHINE_GUID)) {
- if(!dictionary_del(rrdhost_root_index, host->machine_guid))
+ if(!dictionary_del(rrdb.rrdhost_root_index, host->machine_guid))
netdata_log_error("RRDHOST: %s() failed to delete machine guid '%s' from index",
__FUNCTION__, host->machine_guid);
@@ -138,16 +107,16 @@ static void rrdhost_index_del_by_guid(RRDHOST *host) {
inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) {
if(unlikely(!strcmp(hostname, "localhost")))
- return localhost;
+ return rrdb.localhost;
- return dictionary_get(rrdhost_root_index_hostname, hostname);
+ return dictionary_get(rrdb.rrdhost_root_index_hostname, hostname);
}
static inline void rrdhost_index_del_hostname(RRDHOST *host) {
if(unlikely(!host->hostname)) return;
if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_HOSTNAME)) {
- if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)))
+ if(!dictionary_del(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host)))
netdata_log_error("RRDHOST: %s() failed to delete hostname '%s' from index",
__FUNCTION__, rrdhost_hostname(host));
@@ -158,7 +127,7 @@ static inline void rrdhost_index_del_hostname(RRDHOST *host) {
static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
if(!host->hostname) return host;
- RRDHOST *ret_hostname = dictionary_set(rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
+ RRDHOST *ret_hostname = dictionary_set(rrdb.rrdhost_root_index_hostname, rrdhost_hostname(host), host, sizeof(RRDHOST));
if(ret_hostname == host)
rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
else {
@@ -226,14 +195,14 @@ static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, co
host->utc_offset = utc_offset;
}
-void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode,
+void set_host_properties(RRDHOST *host, int update_every, STORAGE_ENGINE_ID storage_engine_id,
const char *registry_hostname, const char *os, const char *tags,
const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *program_name,
const char *program_version)
{
- host->rrd_update_every = update_every;
- host->rrd_memory_mode = memory_mode;
+ host->update_every = update_every;
+ host->storage_engine_id = storage_engine_id;
rrdhost_init_os(host, os);
rrdhost_init_timezone(host, tzone, abbrev_tzone, utc_offset);
@@ -277,7 +246,7 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
rrdhost_option_clear(host, RRDHOST_OPTION_SENDER_ENABLED);
}
-static RRDHOST *rrdhost_create(
+RRDHOST *rrdhost_create(
const char *hostname,
const char *registry_hostname,
const char *guid,
@@ -290,7 +259,7 @@ static RRDHOST *rrdhost_create(
const char *program_version,
int update_every,
long entries,
- RRD_MEMORY_MODE memory_mode,
+ STORAGE_ENGINE_ID storage_engine_id,
unsigned int health_enabled,
unsigned int rrdpush_enabled,
char *rrdpush_destination,
@@ -305,31 +274,30 @@ static RRDHOST *rrdhost_create(
) {
netdata_log_debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
- if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) {
- netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'",
- hostname);
- memory_mode = RRD_MEMORY_MODE_ALLOC;
+ if(storage_engine_id == STORAGE_ENGINE_DBENGINE && !rrdb.dbengine_enabled) {
+ netdata_log_error("memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname);
+ storage_engine_id = STORAGE_ENGINE_ALLOC;
}
#ifdef ENABLE_DBENGINE
- int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);
+ int is_legacy = (storage_engine_id == STORAGE_ENGINE_DBENGINE) && is_legacy_child(guid);
#else
int is_legacy = 1;
#endif
- int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
+ int is_in_multihost = (storage_engine_id == STORAGE_ENGINE_DBENGINE && !is_legacy);
RRDHOST *host = callocz(1, sizeof(RRDHOST));
__atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
strncpyz(host->machine_guid, guid, GUID_LEN + 1);
- set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os,
+ set_host_properties(host, (update_every > 0)?update_every:1, storage_engine_id, registry_hostname, os,
tags, timezone, abbrev_timezone, utc_offset, program_name, program_version);
rrdhost_init_hostname(host, hostname, false);
- host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
- host->health.health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
+ host->rrd_history_entries = align_entries_to_pagesize(storage_engine_id, entries);
+ host->health.health_enabled = ((storage_engine_id == STORAGE_ENGINE_NONE)) ? 0 : health_enabled;
if (likely(!archived)) {
rrdfunctions_init(host);
@@ -347,17 +315,16 @@ int is_legacy = 1;
host->rrdpush_replication_step = rrdpush_replication_step;
host->rrdpush_receiver_replication_percent = 100.0;
- switch(memory_mode) {
- default:
- case RRD_MEMORY_MODE_ALLOC:
- case RRD_MEMORY_MODE_MAP:
- case RRD_MEMORY_MODE_SAVE:
- case RRD_MEMORY_MODE_RAM:
- if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every)
- host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every;
+ switch(storage_engine_id) {
+ case STORAGE_ENGINE_DBENGINE:
break;
-
- case RRD_MEMORY_MODE_DBENGINE:
+ case STORAGE_ENGINE_ALLOC:
+ case STORAGE_ENGINE_MAP:
+ case STORAGE_ENGINE_SAVE:
+ case STORAGE_ENGINE_RAM:
+ default:
+ if(host->rrdpush_seconds_to_replicate > (time_t) host->rrd_history_entries * (time_t) host->update_every)
+ host->rrdpush_seconds_to_replicate = (time_t) host->rrd_history_entries * (time_t) host->update_every;
break;
}
@@ -387,8 +354,8 @@ int is_legacy = 1;
host->cache_dir = strdupz(filename);
}
- if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||
- (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
+ if((host->storage_engine_id == STORAGE_ENGINE_MAP || host->storage_engine_id == STORAGE_ENGINE_SAVE ||
+ (host->storage_engine_id == STORAGE_ENGINE_DBENGINE && is_legacy))) {
int r = mkdir(host->cache_dir, 0775);
if(r != 0 && errno != EEXIST)
netdata_log_error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
@@ -408,7 +375,7 @@ int is_legacy = 1;
rrdcalctemplate_index_init(host);
rrdcalc_rrdhost_index_init(host);
- if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ if (host->storage_engine_id == STORAGE_ENGINE_DBENGINE) {
#ifdef ENABLE_DBENGINE
char dbenginepath[FILENAME_MAX + 1];
int ret;
@@ -424,35 +391,29 @@ int is_legacy = 1;
if (is_legacy) {
// initialize legacy dbengine instance as needed
- host->db[0].mode = RRD_MEMORY_MODE_DBENGINE;
- host->db[0].eng = storage_engine_get(host->db[0].mode);
+ host->db[0].id = STORAGE_ENGINE_DBENGINE;
host->db[0].tier_grouping = get_tier_grouping(0);
- ret = rrdeng_init(
- (struct rrdengine_instance **)&host->db[0].instance,
- dbenginepath,
- default_rrdeng_disk_quota_mb,
- 0); // may fail here for legacy dbengine initialization
+ // may fail here for legacy dbengine initialization
+ ret = rrdeng_init(&host->db[0].instance, dbenginepath, rrdb.default_rrdeng_disk_quota_mb, 0);
if(ret == 0) {
- rrdeng_readiness_wait((struct rrdengine_instance *)host->db[0].instance);
+ rrdeng_readiness_wait(host->db[0].instance);
// assign the rest of the shared storage instances to it
// to allow them collect its metrics too
- for(size_t tier = 1; tier < storage_tiers ; tier++) {
- host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
- host->db[tier].eng = storage_engine_get(host->db[tier].mode);
- host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
+ for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
+ host->db[tier].id= STORAGE_ENGINE_DBENGINE;
+ host->db[tier].instance = rrdb.multidb_ctx[tier];
host->db[tier].tier_grouping = get_tier_grouping(tier);
}
}
}
else {
- for(size_t tier = 0; tier < storage_tiers ; tier++) {
- host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
- host->db[tier].eng = storage_engine_get(host->db[tier].mode);
- host->db[tier].instance = (STORAGE_INSTANCE *)multidb_ctx[tier];
+ for(size_t tier = 0; tier < rrdb.storage_tiers ; tier++) {
+ host->db[tier].id = STORAGE_ENGINE_DBENGINE;
+ host->db[tier].instance = rrdb.multidb_ctx[tier];
host->db[tier].tier_grouping = get_tier_grouping(tier);
}
}
@@ -469,21 +430,19 @@ int is_legacy = 1;
}
#else
- fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");
+ fatal("STORAGE_ENGINE_DBENGINE is not supported in this platform.");
#endif
}
else {
- host->db[0].mode = host->rrd_memory_mode;
- host->db[0].eng = storage_engine_get(host->db[0].mode);
+ host->db[0].id = host->storage_engine_id;
host->db[0].instance = NULL;
host->db[0].tier_grouping = get_tier_grouping(0);
#ifdef ENABLE_DBENGINE
// the first tier is reserved for the non-dbengine modes
- for(size_t tier = 1; tier < storage_tiers ; tier++) {
- host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;
- host->db[tier].eng = storage_engine_get(host->db[tier].mode);
- host->db[tier].instance = (STORAGE_INSTANCE *) multidb_ctx[tier];
+ for(size_t tier = 1; tier < rrdb.storage_tiers ; tier++) {
+ host->db[tier].id = STORAGE_ENGINE_DBENGINE;
+ host->db[tier].instance = rrdb.multidb_ctx[tier];
host->db[tier].tier_grouping = get_tier_grouping(tier);
}
#endif
@@ -517,9 +476,9 @@ int is_legacy = 1;
rrdhost_index_add_hostname(host);
if(is_localhost)
- DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(localhost, host, prev, next);
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
else
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(localhost, host, prev, next);
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
rrd_unlock();
@@ -548,8 +507,8 @@ int is_legacy = 1;
, rrdhost_tags(host)
, rrdhost_program_name(host)
, rrdhost_program_version(host)
- , host->rrd_update_every
- , rrd_memory_mode_name(host->rrd_memory_mode)
+ , host->update_every
+ , storage_engine_name(host->storage_engine_id)
, host->rrd_history_entries
, rrdhost_has_rrdpush_sender_enabled(host)?"enabled":"disabled"
, host->rrdpush_send_destination?host->rrdpush_send_destination:""
@@ -584,7 +543,7 @@ static void rrdhost_update(RRDHOST *host
, const char *program_version
, int update_every
, long history
- , RRD_MEMORY_MODE mode
+ , STORAGE_ENGINE_ID storage_engine_id
, unsigned int health_enabled
, unsigned int rrdpush_enabled
, char *rrdpush_destination
@@ -600,7 +559,7 @@ static void rrdhost_update(RRDHOST *host
spinlock_lock(&host->rrdhost_update_lock);
- host->health.health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
+ host->health.health_enabled = (storage_engine_id == STORAGE_ENGINE_NONE) ? 0 : health_enabled;
{
struct rrdhost_system_info *old = host->system_info;
@@ -636,24 +595,20 @@ static void rrdhost_update(RRDHOST *host
string_freez(t);
}
- if(host->rrd_update_every != update_every)
+ if(host->update_every != update_every)
netdata_log_error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. "
"Restart netdata here to apply the new settings.",
- rrdhost_hostname(host), host->rrd_update_every, update_every);
+ rrdhost_hostname(host), host->update_every, update_every);
- if(host->rrd_memory_mode != mode)
+ if(host->storage_engine_id != storage_engine_id) {
netdata_log_error("Host '%s' has memory mode '%s', but the wanted one is '%s'. "
"Restart netdata here to apply the new settings.",
- rrdhost_hostname(host),
- rrd_memory_mode_name(host->rrd_memory_mode),
- rrd_memory_mode_name(mode));
-
- else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history)
+ rrdhost_hostname(host), storage_engine_name(host->storage_engine_id), storage_engine_name(storage_engine_id));
+ } else if(host->storage_engine_id != STORAGE_ENGINE_DBENGINE && host->rrd_history_entries < history) {
netdata_log_error("Host '%s' has history of %d entries, but the wanted one is %ld entries. "
"Restart netdata here to apply the new settings.",
- rrdhost_hostname(host),
- host->rrd_history_entries,
- history);
+ rrdhost_hostname(host), host->rrd_history_entries, history);
+ }
// update host tags
rrdhost_init_tags(host, tags);
@@ -699,7 +654,7 @@ static void rrdhost_update(RRDHOST *host
spinlock_unlock(&host->rrdhost_update_lock);
}
-RRDHOST *rrdhost_find_or_create(
+RRDHOST *rrdhost_get_or_create(
const char *hostname
, const char *registry_hostname
, const char *guid
@@ -712,7 +667,7 @@ RRDHOST *rrdhost_find_or_create(
, const char *program_version
, int update_every
, long history
- , RRD_MEMORY_MODE mode
+ , STORAGE_ENGINE_ID storage_engine_id
, unsigned int health_enabled
, unsigned int rrdpush_enabled
, char *rrdpush_destination
@@ -727,16 +682,16 @@ RRDHOST *rrdhost_find_or_create(
netdata_log_debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
RRDHOST *host = rrdhost_find_by_guid(guid);
- if (unlikely(host && host->rrd_memory_mode != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
+ if (unlikely(host && host->storage_engine_id != storage_engine_id && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
if (likely(!archived && rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
return host;
- /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
+ /* If a legacy storage engine instantiates all dbengine state must be discarded to avoid inconsistencies */
netdata_log_error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
rrdhost_hostname(host),
- rrd_memory_mode_name(host->rrd_memory_mode),
- rrd_memory_mode_name(mode));
+ storage_engine_name(host->storage_engine_id),
+ storage_engine_name(storage_engine_id));
rrd_wrlock();
rrdhost_free___while_having_rrd_wrlock(host, true);
@@ -758,7 +713,7 @@ RRDHOST *rrdhost_find_or_create(
, program_version
, update_every
, history
- , mode
+ , storage_engine_id
, health_enabled
, rrdpush_enabled
, rrdpush_destination
@@ -787,7 +742,7 @@ RRDHOST *rrdhost_find_or_create(
, program_version
, update_every
, history
- , mode
+ , storage_engine_id
, health_enabled
, rrdpush_enabled
, rrdpush_destination
@@ -804,285 +759,20 @@ RRDHOST *rrdhost_find_or_create(
inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s) {
if(host != protected_host
- && host != localhost
+ && host != rrdb.localhost
&& rrdhost_receiver_replicating_charts(host) == 0
&& rrdhost_sender_replicating_charts(host) == 0
&& rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)
&& !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)
&& !host->receiver
&& host->child_disconnected_time
- && host->child_disconnected_time + rrdhost_free_orphan_time_s < now_s)
+ && host->child_disconnected_time + rrdb.rrdhost_free_orphan_time_s < now_s)
return 1;
return 0;
}
// ----------------------------------------------------------------------------
-// RRDHOST global / startup initialization
-
-#ifdef ENABLE_DBENGINE
-struct dbengine_initialization {
- netdata_thread_t thread;
- char path[FILENAME_MAX + 1];
- int disk_space_mb;
- size_t tier;
- int ret;
-};
-
-void *dbengine_tier_init(void *ptr) {
- struct dbengine_initialization *dbi = ptr;
- dbi->ret = rrdeng_init(NULL, dbi->path, dbi->disk_space_mb, dbi->tier);
- return ptr;
-}
-#endif
-
-void dbengine_init(char *hostname) {
-#ifdef ENABLE_DBENGINE
- use_direct_io = config_get_boolean(CONFIG_SECTION_DB, "dbengine use direct io", use_direct_io);
-
- unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
- if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
- rrdeng_pages_per_extent = read_num;
- else {
- netdata_log_error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
- config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
- }
-
- storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
- if(storage_tiers < 1) {
- netdata_log_error("At least 1 storage tier is required. Assuming 1.");
- storage_tiers = 1;
- config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
- }
- if(storage_tiers > RRD_STORAGE_TIERS) {
- netdata_log_error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS);
- storage_tiers = RRD_STORAGE_TIERS;
- config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
- }
-
- bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
- parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
-
- struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
-
- size_t created_tiers = 0;
- char dbenginepath[FILENAME_MAX + 1];
- char dbengineconfig[200 + 1];
- int divisor = 1;
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if(tier == 0)
- snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
- else
- snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
-
- int ret = mkdir(dbenginepath, 0775);
- if (ret != 0 && errno != EEXIST) {
- netdata_log_error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
- break;
- }
-
- if(tier > 0)
- divisor *= 2;
-
- int disk_space_mb = default_multidb_disk_quota_mb / divisor;
- size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
- RRD_BACKFILL backfill = storage_tiers_backfill[tier];
-
- if(tier > 0) {
- snprintfz(dbengineconfig, 200, "dbengine tier %zu multihost disk space MB", tier);
- disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb);
-
- snprintfz(dbengineconfig, 200, "dbengine tier %zu update every iterations", tier);
- grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
- if(grouping_iterations < 2) {
- grouping_iterations = 2;
- config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
- netdata_log_error("DBENGINE on '%s': 'dbegnine tier %zu update every iterations' cannot be less than 2. Assuming 2.",
- hostname,
- tier);
- }
-
- snprintfz(dbengineconfig, 200, "dbengine tier %zu backfill", tier);
- const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none");
- if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW;
- else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL;
- else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE;
- else {
- netdata_log_error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf);
- config_set(CONFIG_SECTION_DB, dbengineconfig, "new");
- backfill = RRD_BACKFILL_NEW;
- }
- }
-
- storage_tiers_grouping_iterations[tier] = grouping_iterations;
- storage_tiers_backfill[tier] = backfill;
-
- if(tier > 0 && get_tier_grouping(tier) > 65535) {
- storage_tiers_grouping_iterations[tier] = 1;
- netdata_log_error("DBENGINE on '%s': dbengine tier %zu gives aggregation of more than 65535 points of tier 0. Disabling tiers above %zu",
- hostname,
- tier,
- tier);
- break;
- }
-
- internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);
-
- tiers_init[tier].disk_space_mb = disk_space_mb;
- tiers_init[tier].tier = tier;
- strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
- tiers_init[tier].ret = 0;
-
- if(parallel_initialization) {
- char tag[NETDATA_THREAD_TAG_MAX + 1];
- snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);
- netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,
- dbengine_tier_init, &tiers_init[tier]);
- }
- else
- dbengine_tier_init(&tiers_init[tier]);
- }
-
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
- void *ptr;
-
- if(parallel_initialization)
- netdata_thread_join(tiers_init[tier].thread, &ptr);
-
- if(tiers_init[tier].ret != 0) {
- netdata_log_error("DBENGINE on '%s': Failed to initialize multi-host database tier %zu on path '%s'",
- hostname,
- tiers_init[tier].tier,
- tiers_init[tier].path);
- }
- else if(created_tiers == tier)
- created_tiers++;
- }
-
- if(created_tiers && created_tiers < storage_tiers) {
- netdata_log_error("DBENGINE on '%s': Managed to create %zu tiers instead of %zu. Continuing with %zu available.",
- hostname,
- created_tiers,
- storage_tiers,
- created_tiers);
- storage_tiers = created_tiers;
- }
- else if(!created_tiers)
- fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
-
- for(size_t tier = 0; tier < storage_tiers ;tier++)
- rrdeng_readiness_wait(multidb_ctx[tier]);
-
- dbengine_enabled = true;
-#else
- storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
- if(storage_tiers != 1) {
- netdata_log_error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
- storage_tiers = 1;
- config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
- }
- dbengine_enabled = false;
-#endif
-}
-
-int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
- rrdhost_init();
-
- if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
- if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
- set_late_global_environment(system_info);
- fatal("Failed to initialize SQLite");
- }
- netdata_log_info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
- }
-
- if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
- error_report("Failed to initialize context metadata database");
- }
-
- if (unlikely(unittest)) {
- dbengine_enabled = true;
- }
- else {
- health_init();
- rrdpush_init();
-
- if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
- netdata_log_info("DBENGINE: Initializing ...");
- dbengine_init(hostname);
- }
- else {
- netdata_log_info("DBENGINE: Not initializing ...");
- storage_tiers = 1;
- }
-
- if (!dbengine_enabled) {
- if (storage_tiers > 1) {
- netdata_log_error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
- storage_tiers);
- storage_tiers = 1;
- }
-
- if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
- netdata_log_error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
- default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
- }
- }
- }
-
- if(!unittest)
- metadata_sync_init();
-
- netdata_log_debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
- localhost = rrdhost_create(
- hostname
- , registry_get_this_machine_hostname()
- , registry_get_this_machine_guid()
- , os_type
- , netdata_configured_timezone
- , netdata_configured_abbrev_timezone
- , netdata_configured_utc_offset
- , ""
- , program_name
- , program_version
- , default_rrd_update_every
- , default_rrd_history_entries
- , default_rrd_memory_mode
- , default_health_enabled
- , default_rrdpush_enabled
- , default_rrdpush_destination
- , default_rrdpush_api_key
- , default_rrdpush_send_charts_matching
- , default_rrdpush_enable_replication
- , default_rrdpush_seconds_to_replicate
- , default_rrdpush_replication_step
- , system_info
- , 1
- , 0
- );
-
- if (unlikely(!localhost)) {
- return 1;
- }
-
-#ifdef NETDATA_DEV_MODE
- // we register this only on localhost
- // for the other nodes, the origin server should register it
- rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
- rrd_collector_add_function(localhost, NULL, "streaming", 10,
- RRDFUNCTIONS_STREAMING_HELP, true,
- rrdhost_function_streaming, NULL);
-#endif
-
- if (likely(system_info)) {
- migrate_localhost(&localhost->host_uuid);
- sql_aclk_sync_init();
- web_client_api_v1_management_init();
- }
- return localhost==NULL;
-}
-
-// ----------------------------------------------------------------------------
// RRDHOST - free
void rrdhost_system_info_free(struct rrdhost_system_info *system_info) {
@@ -1184,7 +874,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
rrdhost_index_del_by_guid(host);
if (host->prev)
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(localhost, host, prev, next);
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(rrdb.localhost, host, prev, next);
}
// ------------------------------------------------------------------------
@@ -1205,11 +895,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
// release its children resources
#ifdef ENABLE_DBENGINE
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
+ for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
+ if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
&& host->db[tier].instance
&& !is_storage_engine_shared(host->db[tier].instance))
- rrdeng_prepare_exit((struct rrdengine_instance *)host->db[tier].instance);
+ rrdeng_prepare_exit(host->db[tier].instance);
}
#endif
@@ -1226,11 +916,11 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
health_alarm_log_free(host);
#ifdef ENABLE_DBENGINE
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE
+ for(size_t tier = 0; tier < rrdb.storage_tiers ;tier++) {
+ if(host->db[tier].id == STORAGE_ENGINE_DBENGINE
&& host->db[tier].instance
&& !is_storage_engine_shared(host->db[tier].instance))
- rrdeng_exit((struct rrdengine_instance *)host->db[tier].instance);
+ rrdeng_exit(host->db[tier].instance);
}
#endif
@@ -1267,7 +957,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
rrdfamily_index_destroy(host);
rrdfunctions_destroy(host);
rrdvariables_destroy(host->rrdvars);
- if (host == localhost)
+ if (host == rrdb.localhost)
rrdvariables_destroy(health_rrdvars);
rrdhost_destroy_rrdcontexts(host);
@@ -1281,18 +971,18 @@ void rrdhost_free_all(void) {
rrd_wrlock();
/* Make sure child-hosts are released before the localhost. */
- while(localhost && localhost->next)
- rrdhost_free___while_having_rrd_wrlock(localhost->next, true);
+ while (rrdb.localhost && rrdb.localhost->next)
+ rrdhost_free___while_having_rrd_wrlock(rrdb.localhost->next, true);
- if(localhost)
- rrdhost_free___while_having_rrd_wrlock(localhost, true);
+ if (rrdb.localhost)
+ rrdhost_free___while_having_rrd_wrlock(rrdb.localhost, true);
rrd_unlock();
}
void rrd_finalize_collection_for_all_hosts(void) {
RRDHOST *host;
- dfe_start_reentrant(rrdhost_root_index, host) {
+ dfe_start_reentrant(rrdb.rrdhost_root_index, host) {
rrdhost_finalize_collection(host);
}
dfe_done(host);
@@ -1316,181 +1006,22 @@ void rrdhost_save_charts(RRDHOST *host) {
rrdset_foreach_done(st);
}
-struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
- struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
-