diff options
Diffstat (limited to 'database')
24 files changed, 244043 insertions, 2554 deletions
diff --git a/database/engine/Makefile.am b/database/engine/Makefile.am index 90fdc6bac7..43405001d7 100644 --- a/database/engine/Makefile.am +++ b/database/engine/Makefile.am @@ -5,7 +5,6 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in SUBDIRS = \ metadata_log \ - global_uuid_map \ $(NULL) dist_noinst_DATA = \ diff --git a/database/engine/global_uuid_map/Makefile.am b/database/engine/global_uuid_map/Makefile.am deleted file mode 100644 index 161784b8f6..0000000000 --- a/database/engine/global_uuid_map/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) diff --git a/database/engine/global_uuid_map/README.md b/database/engine/global_uuid_map/README.md deleted file mode 100644 index e69de29bb2..0000000000 --- a/database/engine/global_uuid_map/README.md +++ /dev/null diff --git a/database/engine/global_uuid_map/global_uuid_map.c b/database/engine/global_uuid_map/global_uuid_map.c deleted file mode 100644 index 6669517ba1..0000000000 --- a/database/engine/global_uuid_map/global_uuid_map.c +++ /dev/null @@ -1,292 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "global_uuid_map.h" - -static Pvoid_t JGUID_map = (Pvoid_t) NULL; -static Pvoid_t JGUID_object_map = (Pvoid_t) NULL; -static uv_rwlock_t guid_lock; -static uv_rwlock_t object_lock; -static uv_rwlock_t global_lock; - - -void free_global_guid_map() -{ - JudyHSFreeArray(&JGUID_map, PJE0); - JudyHSFreeArray(&JGUID_object_map, PJE0); -} - -static void free_single_uuid(uuid_t *uuid) -{ - Pvoid_t *PValue, *PValue1; - char *existing_object; - Word_t size; - - PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t)); - if (likely(PValue)) { - existing_object = *PValue; - GUID_TYPE object_type = existing_object[0]; - size = (Word_t)object_type ? (object_type * 16) + 1 : strlen((char *)existing_object + 1) + 2; - PValue1 = JudyHSGet(JGUID_object_map, (void *)existing_object, (Word_t)size); - if (PValue1 && *PValue1) { - freez(*PValue1); - } - JudyHSDel(&JGUID_object_map, (void *)existing_object, - (Word_t)object_type ? (object_type * 16) + 1 : strlen((char *)existing_object + 1) + 2, PJE0); - JudyHSDel(&JGUID_map, (void *)uuid, (Word_t)sizeof(uuid_t), PJE0); - freez(existing_object); - } -} - -void free_uuid(uuid_t *uuid) -{ - GUID_TYPE ret; - char object[49]; - - ret = find_object_by_guid(uuid, object, sizeof(object)); - if (GUID_TYPE_DIMENSION == ret) - free_single_uuid((uuid_t *)(object + 16 + 16)); - - if (GUID_TYPE_CHART == ret) - free_single_uuid((uuid_t *)(object + 16)); - - free_single_uuid(uuid); - return; -} - - -void dump_object(uuid_t *index, void *object) -{ - char uuid_s[36 + 1]; - uuid_unparse_lower(*index, uuid_s); - char local_object[3 * 36 + 2 + 1]; - - switch (*(char *) object) { - case GUID_TYPE_CHAR: - debug(D_GUIDLOG, "OBJECT GUID %s on [%s]", uuid_s, (char *)object + 1); - break; - case GUID_TYPE_CHART: - uuid_unparse_lower((const unsigned char *)object + 1, local_object); - uuid_unparse_lower((const unsigned char *)object + 17, local_object+37); - local_object[36] = ':'; - local_object[74] = '\0'; - debug(D_GUIDLOG, "CHART GUID %s on [%s]", uuid_s, local_object); - break; - case GUID_TYPE_DIMENSION: - uuid_unparse_lower((const unsigned char *)object + 1, local_object); - uuid_unparse_lower((const unsigned char *)object + 17, local_object + 37); - uuid_unparse_lower((const unsigned char *)object + 33, local_object + 74); - local_object[36] = ':'; - local_object[73] = ':'; - local_object[110] = '\0'; - debug(D_GUIDLOG, "DIM GUID %s on [%s]", uuid_s, local_object); - break; - default: - debug(D_GUIDLOG, "Unknown object"); - } -} - -/* Returns 0 if it successfully stores the uuid-object mapping or if an identical mapping already exists */ -static inline int guid_store_nolock(uuid_t *uuid, void *object, GUID_TYPE object_type) -{ - char *existing_object; - GUID_TYPE existing_object_type; - - if (unlikely(!object) || uuid == NULL) - return 0; - - Pvoid_t *PValue; - - PValue = JudyHSIns(&JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t), PJE0); - if (PPJERR == PValue) - fatal("JudyHSIns() fatal error."); - if (*PValue) { - existing_object = *PValue; - existing_object_type = existing_object[0]; - if (existing_object_type != object_type) - return 1; - switch (existing_object_type) { - case GUID_TYPE_DIMENSION: - if (memcmp(existing_object, object, 1 + 16 + 16 + 16)) - return 1; - break; - case GUID_TYPE_CHART: - if (memcmp(existing_object, object, 1 + 16 + 16)) - return 1; - break; - case GUID_TYPE_CHAR: - if (strcmp(existing_object + 1, (char *)object)) - return 1; - break; - default: - return 1; - } - freez(existing_object); - } - - *PValue = (Pvoid_t *) object; - - PValue = JudyHSIns(&JGUID_object_map, (void *)object, (Word_t) object_type?(object_type * 16)+1:strlen((char *) object+1)+2, PJE0); - if (PPJERR == PValue) - fatal("JudyHSIns() fatal error."); - if (*PValue == NULL) { - uuid_t *value = (uuid_t *) mallocz(sizeof(uuid_t)); - uuid_copy(*value, *uuid); - *PValue = value; - } - -#ifdef NETDATA_INTERNAL_CHECKS - static uint32_t count = 0; - count++; - char uuid_s[36 + 1]; - uuid_unparse_lower(*uuid, uuid_s); - debug(D_GUIDLOG,"GUID added item %" PRIu32" [%s] as:", count, uuid_s); - dump_object(uuid, object); -#endif - return 0; -} - - -/* - * Given a GUID, find if an object is stored - * - Optionally return the object - */ - -GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes) -{ - Pvoid_t *PValue; - GUID_TYPE value_type; - - uv_rwlock_rdlock(&global_lock); - PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t)); - if (unlikely(!PValue)) { - uv_rwlock_rdunlock(&global_lock); - return GUID_TYPE_NOTFOUND; - } - - value_type = *(char *) *PValue; - - if (likely(object && max_bytes)) { - switch (value_type) { - case GUID_TYPE_CHAR: - if (unlikely(max_bytes - 1 < strlen((char *) *PValue+1))) { - uv_rwlock_rdunlock(&global_lock); - return GUID_TYPE_NOSPACE; - } - strncpyz(object, (char *) *PValue+1, max_bytes - 1); - break; - case GUID_TYPE_HOST: - case GUID_TYPE_CHART: - case GUID_TYPE_DIMENSION: - if (unlikely(max_bytes < (size_t) value_type * 16)) { - uv_rwlock_rdunlock(&global_lock); - return GUID_TYPE_NOSPACE; - } - memcpy(object, *PValue+1, value_type * 16); - break; - default: - uv_rwlock_rdunlock(&global_lock); - return GUID_TYPE_NOTFOUND; - } - } - -#ifdef NETDATA_INTERNAL_CHECKS - dump_object(uuid, *PValue); -#endif - uv_rwlock_rdunlock(&global_lock); - return value_type; -} - -/* - * Find a GUID of an object - * - Optionally return the GUID - * - */ - -int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE object_type) -{ - Pvoid_t *PValue; - - uv_rwlock_rdlock(&global_lock); - PValue = JudyHSGet(JGUID_object_map, (void *)object, (Word_t)object_type?object_type*16+1:strlen(object+1)+2); - if (unlikely(!PValue)) { - uv_rwlock_rdunlock(&global_lock); - return 1; - } - - if (likely(uuid)) - uuid_copy(*uuid, *PValue); - uv_rwlock_rdunlock(&global_lock); - return 0; -} - -int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate) -{ - char *target_object; - uuid_t temp_uuid; - int rc; - - switch (object_type) { - case GUID_TYPE_DIMENSION: - if (unlikely(find_or_generate_guid((void *) ((RRDDIM *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0))) - return 1; - target_object = mallocz(49); - target_object[0] = object_type; - memcpy(target_object + 1, ((RRDDIM *)object)->rrdset->rrdhost->host_uuid, 16); - memcpy(target_object + 17, ((RRDDIM *)object)->rrdset->chart_uuid, 16); - memcpy(target_object + 33, temp_uuid, 16); - break; - case GUID_TYPE_CHART: - if (unlikely(find_or_generate_guid((void *) ((RRDSET *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0))) - return 1; - target_object = mallocz(33); - target_object[0] = object_type; - memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16); - memcpy(target_object + 17, temp_uuid, 16); - break; - case GUID_TYPE_HOST: - target_object = mallocz(17); - target_object[0] = object_type; - memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16); - break; - case GUID_TYPE_CHAR: - target_object = mallocz(strlen((char *) object)+2); - target_object[0] = object_type; - strcpy(target_object+1, (char *) object); - break; - default: - return 1; - } - rc = find_guid_by_object(target_object, uuid, object_type); - if (rc) { - if (!replace_instead_of_generate) /* else take *uuid as user input */ - uuid_generate(*uuid); - uv_rwlock_wrlock(&global_lock); - rc = guid_store_nolock(uuid, target_object, object_type); - uv_rwlock_wrunlock(&global_lock); - if (rc) - freez(target_object); - return rc; - } -#ifdef NETDATA_INTERNAL_CHECKS - dump_object(uuid, target_object); -#endif - freez(target_object); - return 0; -} - -void init_global_guid_map() -{ - static int init = 0; - - if (init) - return; - - init = 1; - info("Configuring locking mechanism for global GUID map"); - fatal_assert(0 == uv_rwlock_init(&guid_lock)); - fatal_assert(0 == uv_rwlock_init(&object_lock)); - fatal_assert(0 == uv_rwlock_init(&global_lock)); - return; -} - - diff --git a/database/engine/global_uuid_map/global_uuid_map.h b/database/engine/global_uuid_map/global_uuid_map.h deleted file mode 100644 index f31f3c0079..0000000000 --- a/database/engine/global_uuid_map/global_uuid_map.h +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_GLOBAL_UUID_MAP_H -#define NETDATA_GLOBAL_UUID_MAP_H - -#include "libnetdata/libnetdata.h" -#include <Judy.h> -#include "../../rrd.h" - -typedef enum guid_type { - GUID_TYPE_CHAR, - GUID_TYPE_HOST, - GUID_TYPE_CHART, - GUID_TYPE_DIMENSION, - GUID_TYPE_NOTFOUND, - GUID_TYPE_NOSPACE -} GUID_TYPE; - -extern GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes); -extern int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE); -extern void init_global_guid_map(); -extern int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate); -extern void free_uuid(uuid_t *uuid); -extern void free_global_guid_map(); -#endif //NETDATA_GLOBAL_UUID_MAP_H diff --git a/database/engine/metadata_log/compaction.c b/database/engine/metadata_log/compaction.c index c261cd9454..ba19e1edfb 100644 --- a/database/engine/metadata_log/compaction.c +++ b/database/engine/metadata_log/compaction.c @@ -3,319 +3,6 @@ #include "metadatalog.h" -void after_compact_old_records(struct metalog_worker_config* wc) -{ - struct metalog_instance *ctx = wc->ctx; - int error; - - mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log, &ctx->compaction_state.new_metadata_logfiles); - uv_run(wc->loop, UV_RUN_DEFAULT); - - error = uv_thread_join(wc->now_compacting_files); - if (error) { - error("uv_thread_join(): %s", uv_strerror(error)); - } - freez(wc->now_compacting_files); - /* unfreeze command processing */ - wc->now_compacting_files = NULL; - - wc->cleanup_thread_compacting_files = 0; - - /* interrupt event loop */ - uv_stop(wc->loop); - - info("Finished metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id); -} - -static void metalog_flush_compaction_records(struct metalog_instance *ctx) -{ - struct metalog_cmd cmd; - struct completion compaction_completion; - - init_completion(&compaction_completion); - - cmd.opcode = METALOG_COMPACTION_FLUSH; - cmd.record_io_descr.completion = &compaction_completion; - metalog_enq_cmd(&ctx->worker_config, &cmd); - - wait_for_completion(&compaction_completion); - destroy_completion(&compaction_completion); -} - -/* The caller must have called metalog_flush_compaction_records() before to synchronize and quiesce the event loop. */ -static void compaction_test_quota(struct metalog_worker_config *wc) -{ - struct metalog_instance *ctx = wc->ctx; - struct logfile_compaction_state *compaction_state; - struct metadata_logfile *oldmetalogfile, *newmetalogfile; - unsigned current_size; - int ret; - - compaction_state = &ctx->compaction_state; - newmetalogfile = compaction_state->new_metadata_logfiles.last; - - oldmetalogfile = ctx->metadata_logfiles.first; - - current_size = newmetalogfile->pos; - if (unlikely(current_size >= MAX_METALOGFILE_SIZE && newmetalogfile->starting_fileno < oldmetalogfile->fileno)) { - /* It's safe to finalize the compacted metadata log file and create a new one since it has already replaced - * an older one. */ - - /* Finalize as the immediately previous file than the currently compacted one. */ - ret = rename_metadata_logfile(newmetalogfile, 0, newmetalogfile->fileno - 1); - if (ret < 0) - return; - - ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, - ctx->metadata_logfiles.first->fileno, ctx->metadata_logfiles.first->fileno); - - if (likely(!ret)) { - compaction_state->fileno = ctx->metadata_logfiles.first->fileno; - } - } -} - - -static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid) -{ - GUID_TYPE ret; - RRDSET *st; - RRDDIM *rd; - BUFFER *buffer; - RRDHOST *host = NULL; - - ret = find_object_by_guid(uuid, NULL, 0); - switch (ret) { - case GUID_TYPE_CHAR: - error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR"); - break; - case GUID_TYPE_CHART: - st = metalog_get_chart_from_uuid(ctx, uuid); - if (st) { - if (ctx->current_compaction_id > st->rrdhost->compaction_id) { - error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id); - compact_record_by_uuid(ctx, &st->rrdhost->host_uuid); - } - - if (ctx->current_compaction_id > st->compaction_id) { - st->compaction_id = ctx->current_compaction_id; - buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id); - metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1); - } else { - debug(D_METADATALOG, "Chart has already been compacted, ignoring record."); - } - } else { - debug(D_METADATALOG, "Ignoring nonexistent chart metadata record."); - } - break; - case GUID_TYPE_DIMENSION: - rd = metalog_get_dimension_from_uuid(ctx, uuid); - if (rd) { - if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) { - error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname); - compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid); - } - if (ctx->current_compaction_id > rd->rrdset->compaction_id) { - error("Forcing compaction of CHART %s", rd->rrdset->id); - compact_record_by_uuid(ctx, rd->rrdset->chart_uuid); - } else if (ctx->current_compaction_id > rd->state->compaction_id) { - rd->state->compaction_id = ctx->current_compaction_id; - buffer = metalog_update_dimension_buffer(rd); - metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1); - } else { - debug(D_METADATALOG, "Dimension has already been compacted, ignoring record."); - } - } else { - debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record."); - } - break; - case GUID_TYPE_HOST: - host = metalog_get_host_from_uuid(ctx, uuid); - if (unlikely(!host)) - break; - if (ctx->current_compaction_id > host->compaction_id) { - host->compaction_id = ctx->current_compaction_id; - buffer = metalog_update_host_buffer(host); - metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1); - } else { - debug(D_METADATALOG, "Host has already been compacted, ignoring record."); - } - break; - case GUID_TYPE_NOTFOUND: - debug(D_METADATALOG, "Ignoring nonexistent metadata record."); - break; - case GUID_TYPE_NOSPACE: - error_with_guid(uuid, "Not enough space for object retrieval"); - break; - default: - error("Unknown return code %u from find_object_by_guid", ret); - break; - } -} - -/* Returns 0 on success. */ -static int compact_metadata_logfile_records(struct metalog_instance *ctx, struct metadata_logfile *metalogfile) -{ - struct metalog_worker_config* wc = &ctx->worker_config; - struct logfile_compaction_state *compaction_state; - struct metalog_record *record; - struct metalog_record_block *record_block, *prev_record_block; - int ret; - unsigned iterated_records; -#define METADATA_LOG_RECORD_BATCH 128 /* Flush I/O and check sizes whenever this many records have been iterated */ - - info("Compacting metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno); - - compaction_state = &ctx->compaction_state; - record_block = prev_record_block = NULL; - iterated_records = 0; - for (record = mlf_record_get_first(metalogfile) ; record != NULL ; record = mlf_record_get_next(metalogfile)) { - if ((record_block = metalogfile->records.iterator.current) != prev_record_block) { - if (prev_record_block) { /* Deallocate iterated record blocks */ - rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr); - freez(prev_record_block); - } - prev_record_block = record_block; - } - compact_record_by_uuid(ctx, &record->uuid); - if (0 == ++iterated_records % METADATA_LOG_RECORD_BATCH) { - metalog_flush_compaction_records(ctx); - if (compaction_state->throttle) { - (void)sleep_usec(10000); /* 10 msec throttle compaction */ - } - compaction_test_quota(wc); - } - } - if (prev_record_block) { /* Deallocate iterated record blocks */ - rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr); - freez(prev_record_block); - } - - info("Compacted metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno); - - metadata_logfile_list_delete(&ctx->metadata_logfiles, metalogfile); - ret = destroy_metadata_logfile(metalogfile); - if (!ret) { - info("Deleted file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno); - rrd_atomic_fetch_add(&ctx->disk_space, -metalogfile->pos); - } else { - error("Failed to delete file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno); - } - freez(metalogfile); - - return ret; -} - -static void compact_old_records(void *arg) -{ - struct metalog_instance *ctx = arg; - struct metalog_worker_config* wc = &ctx->worker_config; - struct logfile_compaction_state *compaction_state; - struct metadata_logfile *metalogfile, *nextmetalogfile, *newmetalogfile; - int ret; - - compaction_state = &ctx->compaction_state; - - nextmetalogfile = NULL; - for (metalogfile = ctx->metadata_logfiles.first ; - metalogfile != compaction_state->last_original_logfile ; - metalogfile = nextmetalogfile) { - nextmetalogfile = metalogfile->next; - - newmetalogfile = compaction_state->new_metadata_logfiles.last; - ret = rename_metadata_logfile(newmetalogfile, newmetalogfile->starting_fileno, metalogfile->fileno); - if (ret < 0) { - error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno); - } - - ret = compact_metadata_logfile_records(ctx, metalogfile); - if (ret) { - error("Metadata log compaction failed, cancelling."); - break; - } - } - fatal_assert(nextmetalogfile); /* There are always more than 1 metadata log files during compaction */ - - newmetalogfile = compaction_state->new_metadata_logfiles.last; - if (newmetalogfile->starting_fileno != 0) { /* Must rename the last compacted file */ - ret = rename_metadata_logfile(newmetalogfile, 0, nextmetalogfile->fileno - 1); - if (ret < 0) { - error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".", - ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno); - } - } - /* Connect the compacted files to the metadata log */ - newmetalogfile->next = nextmetalogfile; - ctx->metadata_logfiles.first = compaction_state->new_metadata_logfiles.first; - - wc->cleanup_thread_compacting_files = 1; - /* wake up event loop */ - fatal_assert(0 == uv_async_send(&wc->async)); -} - -/* Returns 0 on success. */ -static int init_compaction_state(struct metalog_instance *ctx) -{ - struct metadata_logfile *newmetalogfile; - struct logfile_compaction_state *compaction_state; - int ret; - - compaction_state = &ctx->compaction_state; - compaction_state->new_metadata_logfiles.first = NULL; - compaction_state->new_metadata_logfiles.last = NULL; - compaction_state->starting_fileno = ctx->metadata_logfiles.first->fileno; - compaction_state->fileno = ctx->metadata_logfiles.first->fileno; - compaction_state->last_original_logfile = ctx->metadata_logfiles.last; - compaction_state->throttle = 0; - - ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, compaction_state->starting_fileno, - compaction_state->fileno); - if (unlikely(ret)) { - error("Cannot create new metadata log files, compaction aborted."); - return ret; - } - newmetalogfile = compaction_state->new_metadata_logfiles.first; - fatal_assert(newmetalogfile == compaction_state->new_metadata_logfiles.last); - init_metadata_record_log(&compaction_state->records_log); - - return 0; -} - -void metalog_do_compaction(struct metalog_worker_config *wc) -{ - struct metalog_instance *ctx = wc->ctx; - int error; - - if (wc->now_compacting_files) { - /* already compacting metadata log files */ - return; - } - wc->now_compacting_files = mallocz(sizeof(*wc->now_compacting_files)); - wc->cleanup_thread_compacting_files = 0; - metalog_try_link_new_metadata_logfile(wc); - - error = init_compaction_state(ctx); - if (unlikely(error)) { - error("Cannot create new metadata log files, compaction aborted."); - return; - } - ++ctx->current_compaction_id; /* Signify a new compaction */ - - info("Starting metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id); - error = uv_thread_create(wc->now_compacting_files, compact_old_records, ctx); - if (error) { - error("uv_thread_create(): %s", uv_strerror(error)); - freez(wc->now_compacting_files); - wc->now_compacting_files = NULL; - } - -} - /* Return 0 on success. */ int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles, unsigned *matched_files) diff --git a/database/engine/metadata_log/compaction.h b/database/engine/metadata_log/compaction.h index da4765eeb5..d046134403 100644 --- a/database/engine/metadata_log/compaction.h +++ b/database/engine/metadata_log/compaction.h @@ -8,19 +8,7 @@ #endif #include "../rrdengine.h" -struct logfile_compaction_state { - unsigned fileno; /* Starts at 1 */ - unsigned starting_fileno; /* 0 for normal files, staring number during compaction */ - - struct metadata_record_commit_log records_log; - struct metadata_logfile_list new_metadata_logfiles; - struct metadata_logfile *last_original_logfile; /* Marks the end of compaction */ - uint8_t throttle; /* set non-zero to throttle compaction */ -}; - extern int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles, unsigned *matched_files); -extern void metalog_do_compaction(struct metalog_worker_config *wc); -extern void after_compact_old_records(struct metalog_worker_config* wc); #endif /* NETDATA_COMPACTION_H */ diff --git a/database/engine/metadata_log/logfile.c b/database/engine/metadata_log/logfile.c index 08bb4eec86..b7c5c06182 100644 --- a/database/engine/metadata_log/logfile.c +++ b/database/engine/metadata_log/logfile.c @@ -1,184 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include <database/sqlite/sqlite_functions.h> #include "metadatalog.h" #include "metalogpluginsd.h" -static void mlf_record_block_insert(struct metadata_logfile *metalogfile, struct metalog_record_block *record_block) -{ - - if (likely(NULL != metalogfile->records.last)) { - metalogfile->records.last->next = record_block; - } - if (unlikely(NULL == metalogfile->records.first)) { - metalogfile->records.first = record_block; - } |