diff options
author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2024-04-03 17:02:25 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-04-03 17:02:25 +0300 |
commit | 0cec40e5252f8a7b1f68ff8ab5a1c4cbf1d8afe6 (patch) | |
tree | 520edcd5c9bd26a8c10b31c45677f5f4d21402bd | |
parent | 1549626cac4bfbf3102c013e55c19e92ac4b2a8b (diff) |
Drop legacy dbengine support (#17315)
Remove legacy dbengine
-rw-r--r-- | src/daemon/global_statistics.c | 10 | ||||
-rw-r--r-- | src/database/engine/rrdengine.c | 3 | ||||
-rw-r--r-- | src/database/engine/rrdengine.h | 2 | ||||
-rwxr-xr-x | src/database/engine/rrdengineapi.c | 53 | ||||
-rw-r--r-- | src/database/engine/rrdengineapi.h | 3 | ||||
-rw-r--r-- | src/database/engine/rrdenginelib.c | 64 | ||||
-rw-r--r-- | src/database/engine/rrdenginelib.h | 1 | ||||
-rw-r--r-- | src/database/rrd.h | 1 | ||||
-rw-r--r-- | src/database/rrdhost.c | 188 |
9 files changed, 101 insertions, 224 deletions
diff --git a/src/daemon/global_statistics.c b/src/daemon/global_statistics.c index 1798bfddb8..7a8fc4c385 100644 --- a/src/daemon/global_statistics.c +++ b/src/daemon/global_statistics.c @@ -2562,12 +2562,10 @@ static void dbengine2_statistics_charts(void) { if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue; if(!host->db[tier].si) continue; - if(is_storage_engine_shared(host->db[tier].si)) { - if(counted_multihost_db[tier]) - continue; - else - counted_multihost_db[tier] = 1; - } + if(counted_multihost_db[tier]) + continue; + else + counted_multihost_db[tier] = 1; ++dbengine_contexts; rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].si, local_stats_array); diff --git a/src/database/engine/rrdengine.c b/src/database/engine/rrdengine.c index 7b21374366..c47a3f2b10 100644 --- a/src/database/engine/rrdengine.c +++ b/src/database/engine/rrdengine.c @@ -1366,8 +1366,7 @@ static void *ctx_shutdown_tp_worker(struct rrdengine_instance *ctx __maybe_unuse if(!logged) { logged = true; netdata_log_info("DBENGINE: waiting for %zu inflight queries to finish to shutdown tier %d...", - __atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED), - (ctx->config.legacy) ? -1 : ctx->config.tier); + __atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED), ctx->config.tier); } sleep_usec(1 * USEC_PER_MS); } diff --git a/src/database/engine/rrdengine.h b/src/database/engine/rrdengine.h index 3047e0c6ac..86cf56c2db 100644 --- a/src/database/engine/rrdengine.h +++ b/src/database/engine/rrdengine.h @@ -350,8 +350,6 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele struct rrdengine_instance { struct { - bool legacy; // true when the db is autonomous for a single host - int tier; // the tier of this ctx uint8_t page_type; // default page type for this context diff --git a/src/database/engine/rrdengineapi.c b/src/database/engine/rrdengineapi.c index 43fed492b5..f596798409 100755 --- a/src/database/engine/rrdengineapi.c +++ b/src/database/engine/rrdengineapi.c @@ -98,26 +98,18 @@ void rrdeng_metrics_group_release(STORAGE_INSTANCE *si __maybe_unused, STORAGE_M // metric handle for legacy dbs /* This UUID is not unique across hosts */ -void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid) +void rrdeng_generate_unittest_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid) { - EVP_MD_CTX *evpctx; - unsigned char hash_value[EVP_MAX_MD_SIZE]; - unsigned int hash_len; - - evpctx = EVP_MD_CTX_create(); - EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL); - EVP_DigestUpdate(evpctx, dim_id, strlen(dim_id)); - EVP_DigestUpdate(evpctx, chart_id, strlen(chart_id)); - EVP_DigestFinal_ex(evpctx, hash_value, &hash_len); - EVP_MD_CTX_destroy(evpctx); - fatal_assert(hash_len > sizeof(uuid_t)); - memcpy(ret_uuid, hash_value, sizeof(uuid_t)); + CLEAN_BUFFER *wb = buffer_create(100, NULL); + buffer_sprintf(wb,"%s.%s", dim_id, chart_id); + UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb)); + uuid_copy(*ret_uuid, uuid.uuid); } -static METRIC *rrdeng_metric_get_legacy(STORAGE_INSTANCE *si, const char *rd_id, const char *st_id) { +static METRIC *rrdeng_metric_unittest(STORAGE_INSTANCE *si, const char *rd_id, const char *st_id) { struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; uuid_t legacy_uuid; - rrdeng_generate_legacy_uuid(rd_id, st_id, &legacy_uuid); + rrdeng_generate_unittest_uuid(rd_id, st_id, &legacy_uuid); return mrg_metric_get_and_acquire(main_mrg, &legacy_uuid, (Word_t) ctx); } @@ -165,11 +157,8 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE metric = mrg_metric_get_and_acquire(main_mrg, &rd->metric_uuid, (Word_t) ctx); if(unlikely(!metric)) { - if(unlikely(ctx->config.legacy)) { - // this is a single host database - // generate uuid from the chart and dimensions ids - // and overwrite the one supplied by rrddim - metric = rrdeng_metric_get_legacy(si, rrddim_id(rd), rrdset_id(rd->rrdset)); + if(unlikely(unittest_running)) { + metric = rrdeng_metric_unittest(si, rrddim_id(rd), rrdset_id(rd->rrdset)); if (metric) uuid_copy(rd->metric_uuid, *mrg_metric_uuid(main_mrg, metric)); } @@ -1128,11 +1117,6 @@ void rrdeng_readiness_wait(struct rrdengine_instance *ctx) { netdata_log_info("DBENGINE: tier %d is ready for data collection and queries", ctx->config.tier); } -bool rrdeng_is_legacy(STORAGE_INSTANCE *si) { - struct rrdengine_instance *ctx = (struct rrdengine_instance *)si; - return ctx->config.legacy; -} - void rrdeng_exit_mode(struct rrdengine_instance *ctx) { __atomic_store_n(&ctx->quiesce.exit_mode, true, __ATOMIC_RELAXED); } @@ -1159,14 +1143,11 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path, return UV_EMFILE; } - if(NULL == ctxp) { + if(ctxp) + *ctxp = ctx = callocz(1, sizeof(*ctx)); + else { ctx = multidb_ctx[tier]; memset(ctx, 0, sizeof(*ctx)); - ctx->config.legacy = false; - } - else { - *ctxp = ctx = callocz(1, sizeof(*ctx)); - ctx->config.legacy = true; } ctx->config.tier = (int)tier; @@ -1192,7 +1173,7 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path, return 0; } - if (ctx->config.legacy) { + if (unittest_running) { freez(ctx); if (ctxp) *ctxp = NULL; @@ -1223,17 +1204,17 @@ int rrdeng_exit(struct rrdengine_instance *ctx) { size_t count = 10; while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && count && !unittest_running) { if(!logged) { - netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier); + netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", ctx->config.tier); logged = true; } sleep_usec(100 * USEC_PER_MS); count--; } - netdata_log_info("DBENGINE: flushing main cache for tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier); + netdata_log_info("DBENGINE: flushing main cache for tier %d", ctx->config.tier); pgc_flush_all_hot_and_dirty_pages(main_cache, (Word_t)ctx); - netdata_log_info("DBENGINE: shutting down tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier); + netdata_log_info("DBENGINE: shutting down tier %d", ctx->config.tier); struct completion completion = {}; completion_init(&completion); rrdeng_enq_cmd(ctx, RRDENG_OPCODE_CTX_SHUTDOWN, NULL, &completion, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL); @@ -1242,7 +1223,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx) { finalize_rrd_files(ctx); - if(ctx->config.legacy) + if (unittest_running) //(ctx->config.unittest) freez(ctx); rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE); diff --git a/src/database/engine/rrdengineapi.h b/src/database/engine/rrdengineapi.h index fb449cd9ba..4d49421b5a 100644 --- a/src/database/engine/rrdengineapi.h +++ b/src/database/engine/rrdengineapi.h @@ -24,8 +24,6 @@ extern uint8_t tier_page_type[]; #define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type] -void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid); - STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si); STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, uuid_t *uuid); void rrdeng_metric_release(STORAGE_METRIC_HANDLE *smh); @@ -221,6 +219,5 @@ struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void); RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx); size_t rrdeng_collectors_running(struct rrdengine_instance *ctx); -bool rrdeng_is_legacy(STORAGE_INSTANCE *si); #endif /* NETDATA_RRDENGINEAPI_H */ diff --git a/src/database/engine/rrdenginelib.c b/src/database/engine/rrdenginelib.c index dc581d98d9..2ca62555eb 100644 --- a/src/database/engine/rrdenginelib.c +++ b/src/database/engine/rrdenginelib.c @@ -76,51 +76,6 @@ int open_file_for_io(char *path, int flags, uv_file *file, int direct) return fd; } -int is_legacy_child(const char *machine_guid) -{ - uuid_t uuid; - char dbengine_file[FILENAME_MAX+1]; - - if (unlikely(!strcmp(machine_guid, "unittest-dbengine") || !strcmp(machine_guid, "dbengine-dataset") || - !strcmp(machine_guid, "dbengine-stress-test"))) { - return 1; - } - if (!uuid_parse(machine_guid, uuid)) { - uv_fs_t stat_req; - snprintfz(dbengine_file, FILENAME_MAX, "%s/%s/dbengine", netdata_configured_cache_dir, machine_guid); - int rc = uv_fs_stat(NULL, &stat_req, dbengine_file, NULL); - if (likely(rc == 0 && ((stat_req.statbuf.st_mode & S_IFMT) == S_IFDIR))) { - //netdata_log_info("Found legacy engine folder \"%s\"", dbengine_file); - return 1; - } - } - return 0; -} - -int count_legacy_children(char *dbfiles_path) -{ - int ret; - uv_fs_t req; - uv_dirent_t dent; - int legacy_engines = 0; - - ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL); - if (ret < 0) { - uv_fs_req_cleanup(&req); - netdata_log_error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret)); - return ret; - } - - while(UV_EOF != uv_fs_scandir_next(&req, &dent)) { - if (dent.type == UV_DIRENT_DIR) { - if (is_legacy_child(dent.name)) - legacy_engines++; - } - } - uv_fs_req_cleanup(&req); - return legacy_engines; -} - int compute_multidb_diskspace() { char multidb_disk_space_file[FILENAME_MAX + 1]; @@ -139,23 +94,8 @@ int compute_multidb_diskspace() } } - if (computed_multidb_disk_quota_mb == -1) { - int rc = count_legacy_children(netdata_configured_cache_dir); - if (likely(rc >= 0)) { - computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb; - netdata_log_info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb); - - fp = fopen(multidb_disk_space_file, "w"); - if (likely(fp)) { - fprintf(fp, "%d", computed_multidb_disk_quota_mb); - netdata_log_info("Created file '%s' to store the computed value", multidb_disk_space_file); - fclose(fp); - } else - netdata_log_error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file); - } - else - computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; - } + if (computed_multidb_disk_quota_mb == -1) + computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; return computed_multidb_disk_quota_mb; } diff --git a/src/database/engine/rrdenginelib.h b/src/database/engine/rrdenginelib.h index a0febd4f4c..cb8c8fd2e7 100644 --- a/src/database/engine/rrdenginelib.h +++ b/src/database/engine/rrdenginelib.h @@ -89,6 +89,5 @@ static inline int open_file_buffered_io(char *path, int flags, uv_file *file) return open_file_for_io(path, flags, file, 0); } int compute_multidb_diskspace(); -int is_legacy_child(const char *machine_guid); #endif /* NETDATA_RRDENGINELIB_H */ diff --git a/src/database/rrd.h b/src/database/rrd.h index 3295f036c3..2c3237401e 100644 --- a/src/database/rrd.h +++ b/src/database/rrd.h @@ -1369,7 +1369,6 @@ extern netdata_rwlock_t rrd_rwlock; // ---------------------------------------------------------------------------- -bool is_storage_engine_shared(STORAGE_INSTANCE *si); void rrdset_index_init(RRDHOST *host); void rrdset_index_destroy(RRDHOST *host); diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c index 9c818618fc..8ea1611bb9 100644 --- a/src/database/rrdhost.c +++ b/src/database/rrdhost.c @@ -33,15 +33,6 @@ time_t rrdset_free_obsolete_time_s = 3600; time_t rrdhost_free_orphan_time_s = 3600; time_t rrdhost_free_ephemeral_time_s = 86400; -bool is_storage_engine_shared(STORAGE_INSTANCE *si __maybe_unused) { -#ifdef ENABLE_DBENGINE - if(!rrdeng_is_legacy(si)) - return true; -#endif - - return false; -} - RRDHOST *find_host_by_node_id(char *node_id) { uuid_t node_uuid; @@ -269,6 +260,70 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host, rrdhost_option_clear(host, RRDHOST_OPTION_SENDER_ENABLED); } +// +// true on success +// +static bool create_dbengine_directory(RRDHOST *host, const char *dbenginepath) +{ + int ret = mkdir(dbenginepath, 0775); + if (ret != 0 && errno != EEXIST) { + nd_log(NDLS_DAEMON, NDLP_CRIT, "Host '%s': cannot create directory '%s'", rrdhost_hostname(host), dbenginepath); + return false; + } + return true; +} + +static RRDHOST *prepare_host_for_unittest(RRDHOST *host) +{ + char dbenginepath[FILENAME_MAX + 1]; + + if (host->cache_dir) + freez(host->cache_dir); + + snprintfz(dbenginepath, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid); + host->cache_dir = strdupz(dbenginepath); + + int ret = 0; + + bool initialized; + if ((initialized = create_dbengine_directory(host, dbenginepath))) { + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir); + + if ((initialized = create_dbengine_directory(host, dbenginepath))) { + host->db[0].mode = RRD_MEMORY_MODE_DBENGINE; + host->db[0].eng = storage_engine_get(host->db[0].mode); + host->db[0].tier_grouping = get_tier_grouping(0); + + ret = rrdeng_init( + (struct rrdengine_instance **)&host->db[0].si, + dbenginepath, + default_rrdeng_disk_quota_mb, + 0); // may fail here for legacy dbengine initialization + + initialized = (ret == 0); + + if (initialized) + rrdeng_readiness_wait((struct rrdengine_instance *)host->db[0].si); + } + } + + if (!initialized) { + nd_log( + NDLS_DAEMON, + NDLP_CRIT, + "Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.", + rrdhost_hostname(host), + host->machine_guid, + host->cache_dir); + + rrd_wrlock(); + rrdhost_free___while_having_rrd_wrlock(host, true); + rrd_unlock(); + return NULL; + } + return host; +} + static RRDHOST *rrdhost_create( const char *hostname, const char *registry_hostname, @@ -302,20 +357,20 @@ static RRDHOST *rrdhost_create( memory_mode = RRD_MEMORY_MODE_ALLOC; } -#ifdef ENABLE_DBENGINE - int is_legacy = (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid); -#else -int is_legacy = 1; -#endif - - int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy); RRDHOST *host = callocz(1, sizeof(RRDHOST)); __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED); strncpyz(host->machine_guid, guid, GUID_LEN + 1); - set_host_properties(host, (update_every > 0)?update_every:1, memory_mode, registry_hostname, os, - timezone, abbrev_timezone, utc_offset, + set_host_properties( + host, + (update_every > 0) ? update_every : 1, + memory_mode, + registry_hostname, + os, + timezone, + abbrev_timezone, + utc_offset, prog_name, prog_version); @@ -360,28 +415,8 @@ int is_legacy = 1; rrdset_index_init(host); - char filename[FILENAME_MAX + 1]; if(is_localhost) host->cache_dir = strdupz(netdata_configured_cache_dir); - else { - // this is not localhost - append our GUID to localhost path - if (is_in_multihost) { // don't append to cache dir in multihost - host->cache_dir = strdupz(netdata_configured_cache_dir); - } - else { - snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid); - host->cache_dir = strdupz(filename); - } - - if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy) - { - int r = mkdir(host->cache_dir, 0775); - if(r != 0 && errno != EEXIST) - nd_log(NDLS_DAEMON, NDLP_CRIT, - "Host '%s': cannot create directory '%s'", - rrdhost_hostname(host), host->cache_dir); - } - } // this is also needed for custom host variables - not only health if(!host->rrdvars) @@ -396,45 +431,10 @@ int is_legacy = 1; if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { #ifdef ENABLE_DBENGINE - char dbenginepath[FILENAME_MAX + 1]; - int ret; - - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir); - ret = mkdir(dbenginepath, 0775); - - if (ret != 0 && errno != EEXIST) - nd_log(NDLS_DAEMON, NDLP_CRIT, - "Host '%s': cannot create directory '%s'", - rrdhost_hostname(host), dbenginepath); - else - ret = 0; // succeed - - if (is_legacy) { - // initialize legacy dbengine instance as needed - - host->db[0].mode = RRD_MEMORY_MODE_DBENGINE; - host->db[0].eng = storage_engine_get(host->db[0].mode); - host->db[0].tier_grouping = get_tier_grouping(0); - - ret = rrdeng_init( - (struct rrdengine_instance **)&host->db[0].si, - dbenginepath, - default_rrdeng_disk_quota_mb, - 0); // may fail here for legacy dbengine initialization - - if(ret == 0) { - rrdeng_readiness_wait((struct rrdengine_instance *)host->db[0].si); - - // assign the rest of the shared storage instances to it - // to allow them collect its metrics too - - for(size_t tier = 1; tier < storage_tiers ; tier++) { - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE; - host->db[tier].eng = storage_engine_get(host->db[tier].mode); - host->db[tier].si = (STORAGE_INSTANCE *) multidb_ctx[tier]; - host->db[tier].tier_grouping = get_tier_grouping(tier); - } - } + if (unittest_running) { + host = prepare_host_for_unittest(host); + if (!host) + return NULL; } else { for(size_t tier = 0; tier < storage_tiers ; tier++) { @@ -444,19 +444,6 @@ int is_legacy = 1; host->db[tier].tier_grouping = get_tier_grouping(tier); } } - - if (ret) { // check legacy or multihost initialization success - nd_log(NDLS_DAEMON, NDLP_CRIT, - "Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.", - rrdhost_hostname(host), host->machine_guid, host->cache_dir); - - rrd_wrlock(); - rrdhost_free___while_having_rrd_wrlock(host, true); - rrd_unlock(); - - return NULL; - } - #else fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform."); #endif @@ -1234,18 +1221,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) { rrdcalc_delete_all(host); - // ------------------------------------------------------------------------ - // release its children resources - -#ifdef ENABLE_DBENGINE - for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE - && host->db[tier].si - && !is_storage_engine_shared(host->db[tier].si)) - rrdeng_prepare_exit((struct rrdengine_instance *)host->db[tier].si); - } -#endif - // delete all the RRDSETs of the host rrdset_index_destroy(host); rrdcalc_rrdhost_index_destroy(host); @@ -1257,15 +1232,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) { health_alarm_log_free(host); -#ifdef ENABLE_DBENGINE - for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(host->db[tier].mode == RRD_MEMORY_MODE_DBENGINE - && host->db[tier].si - && !is_storage_engine_shared(host->db[tier].si)) - rrdeng_exit((struct rrdengine_instance *)host->db[tier].si); - } -#endif - if (!netdata_exit && !force) { nd_log(NDLS_DAEMON, NDLP_DEBUG, "RRD: 'host:%s' is now in archive mode...", |