Migrate metadata log to SQLite (#10139)

24 files changed, 244043 insertions, 2554 deletions

diff --git a/database/engine/Makefile.am b/database/engine/Makefile.am
index 90fdc6bac7..43405001d7 100644
--- a/database/engine/Makefile.am
+++ b/database/engine/Makefile.am
@@ -5,7 +5,6 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
 SUBDIRS = \
     metadata_log \
-    global_uuid_map \
     $(NULL)
 
 dist_noinst_DATA = \
diff --git a/database/engine/global_uuid_map/Makefile.am b/database/engine/global_uuid_map/Makefile.am
deleted file mode 100644
index 161784b8f6..0000000000
--- a/database/engine/global_uuid_map/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
-    README.md \
-    $(NULL)
diff --git a/database/engine/global_uuid_map/README.md b/database/engine/global_uuid_map/README.md
deleted file mode 100644
index e69de29bb2..0000000000
--- a/database/engine/global_uuid_map/README.md
+++ /dev/null
diff --git a/database/engine/global_uuid_map/global_uuid_map.c b/database/engine/global_uuid_map/global_uuid_map.c
deleted file mode 100644
index 6669517ba1..0000000000
--- a/database/engine/global_uuid_map/global_uuid_map.c
+++ /dev/null
@@ -1,292 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "global_uuid_map.h"
-
-static Pvoid_t JGUID_map = (Pvoid_t) NULL;
-static Pvoid_t JGUID_object_map = (Pvoid_t) NULL;
-static uv_rwlock_t guid_lock;
-static uv_rwlock_t object_lock;
-static uv_rwlock_t global_lock;
-
-
-void free_global_guid_map()
-{
-    JudyHSFreeArray(&JGUID_map, PJE0);
-    JudyHSFreeArray(&JGUID_object_map, PJE0);
-}
-
-static void free_single_uuid(uuid_t *uuid)
-{
-    Pvoid_t *PValue, *PValue1;
-    char *existing_object;
-    Word_t  size;
-
-    PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t));
-    if (likely(PValue)) {
-        existing_object = *PValue;
-        GUID_TYPE object_type = existing_object[0];
-        size = (Word_t)object_type ? (object_type * 16) + 1 : strlen((char *)existing_object + 1) + 2;
-        PValue1 = JudyHSGet(JGUID_object_map, (void *)existing_object, (Word_t)size);
-        if (PValue1 && *PValue1) {
-            freez(*PValue1);
-        }
-        JudyHSDel(&JGUID_object_map, (void *)existing_object,
-                  (Word_t)object_type ? (object_type * 16) + 1 : strlen((char *)existing_object + 1) + 2, PJE0);
-        JudyHSDel(&JGUID_map, (void *)uuid, (Word_t)sizeof(uuid_t), PJE0);
-        freez(existing_object);
-    }
-}
-
-void free_uuid(uuid_t *uuid)
-{
-    GUID_TYPE ret;
-    char object[49];
-
-    ret = find_object_by_guid(uuid, object, sizeof(object));
-    if (GUID_TYPE_DIMENSION == ret)
-        free_single_uuid((uuid_t *)(object + 16 + 16));
-
-    if (GUID_TYPE_CHART == ret)
-            free_single_uuid((uuid_t *)(object + 16));
-
-    free_single_uuid(uuid);
-    return;
-}
-
-
-void   dump_object(uuid_t *index, void *object)
-{
-    char uuid_s[36 + 1];
-    uuid_unparse_lower(*index, uuid_s);
-    char local_object[3 * 36 + 2 + 1];
-
-    switch (*(char *) object) {
-        case GUID_TYPE_CHAR:
-            debug(D_GUIDLOG, "OBJECT GUID %s on [%s]", uuid_s, (char *)object + 1);
-            break;
-        case GUID_TYPE_CHART:
-            uuid_unparse_lower((const unsigned char *)object + 1, local_object);
-            uuid_unparse_lower((const unsigned char *)object + 17, local_object+37);
-            local_object[36] = ':';
-            local_object[74] = '\0';
-            debug(D_GUIDLOG, "CHART GUID %s on [%s]", uuid_s, local_object);
-            break;
-        case GUID_TYPE_DIMENSION:
-            uuid_unparse_lower((const unsigned char *)object + 1, local_object);
-            uuid_unparse_lower((const unsigned char *)object + 17, local_object + 37);
-            uuid_unparse_lower((const unsigned char *)object + 33, local_object + 74);
-            local_object[36] = ':';
-            local_object[73] = ':';
-            local_object[110] = '\0';
-            debug(D_GUIDLOG, "DIM GUID %s on [%s]", uuid_s, local_object);
-            break;
-        default:
-            debug(D_GUIDLOG, "Unknown object");
-    }
-}
-
-/* Returns 0 if it successfully stores the uuid-object mapping or if an identical mapping already exists */
-static inline int guid_store_nolock(uuid_t *uuid, void *object, GUID_TYPE object_type)
-{
-    char *existing_object;
-    GUID_TYPE existing_object_type;
-
-    if (unlikely(!object) || uuid == NULL)
-        return 0;
-
-    Pvoid_t *PValue;
-
-    PValue = JudyHSIns(&JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t), PJE0);
-    if (PPJERR == PValue)
-        fatal("JudyHSIns() fatal error.");
-    if (*PValue) {
-        existing_object = *PValue;
-        existing_object_type = existing_object[0];
-        if (existing_object_type != object_type)
-            return 1;
-        switch (existing_object_type) {
-            case GUID_TYPE_DIMENSION:
-                if (memcmp(existing_object, object, 1 + 16 + 16 + 16))
-                    return 1;
-                break;
-            case GUID_TYPE_CHART:
-                if (memcmp(existing_object, object, 1 + 16 + 16))
-                    return 1;
-                break;
-            case GUID_TYPE_CHAR:
-                if (strcmp(existing_object + 1, (char *)object))
-                    return 1;
-                break;
-            default:
-                return 1;
-        }
-        freez(existing_object);
-    }
-
-    *PValue = (Pvoid_t *) object;
-
-    PValue = JudyHSIns(&JGUID_object_map, (void *)object, (Word_t) object_type?(object_type * 16)+1:strlen((char *) object+1)+2, PJE0);
-    if (PPJERR == PValue)
-        fatal("JudyHSIns() fatal error.");
-    if (*PValue == NULL) {
-        uuid_t *value = (uuid_t *) mallocz(sizeof(uuid_t));
-        uuid_copy(*value, *uuid);
-        *PValue = value;
-    }
-
-#ifdef NETDATA_INTERNAL_CHECKS
-    static uint32_t count = 0;
-    count++;
-    char uuid_s[36 + 1];
-    uuid_unparse_lower(*uuid, uuid_s);
-    debug(D_GUIDLOG,"GUID added item %" PRIu32" [%s] as:", count, uuid_s);
-    dump_object(uuid, object);
-#endif
-    return 0;
-}
-
-
-/*
- * Given a GUID, find if an object is stored
- *   - Optionally return the object
- */
-
-GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
-{
-    Pvoid_t *PValue;
-    GUID_TYPE   value_type;
-
-    uv_rwlock_rdlock(&global_lock);
-    PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t));
-    if (unlikely(!PValue)) {
-        uv_rwlock_rdunlock(&global_lock);
-        return GUID_TYPE_NOTFOUND;
-    }
-
-    value_type = *(char *) *PValue;
-
-    if (likely(object && max_bytes)) {
-        switch (value_type) {
-            case GUID_TYPE_CHAR:
-                if (unlikely(max_bytes - 1 < strlen((char *) *PValue+1))) {
-                    uv_rwlock_rdunlock(&global_lock);
-                    return GUID_TYPE_NOSPACE;
-                }
-                strncpyz(object, (char *) *PValue+1, max_bytes - 1);
-                break;
-            case GUID_TYPE_HOST:
-            case GUID_TYPE_CHART:
-            case GUID_TYPE_DIMENSION:
-                if (unlikely(max_bytes < (size_t) value_type * 16)) {
-                    uv_rwlock_rdunlock(&global_lock);
-                    return GUID_TYPE_NOSPACE;
-                }
-                memcpy(object, *PValue+1, value_type * 16);
-                break;
-            default:
-                uv_rwlock_rdunlock(&global_lock);
-                return GUID_TYPE_NOTFOUND;
-        }
-    }
-
-#ifdef NETDATA_INTERNAL_CHECKS
-    dump_object(uuid, *PValue);
-#endif
-    uv_rwlock_rdunlock(&global_lock);
-    return value_type;
-}
-
-/*
- * Find a GUID of an object
- *   - Optionally return the GUID
- *
- */
-
-int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE object_type)
-{
-    Pvoid_t *PValue;
-
-    uv_rwlock_rdlock(&global_lock);
-    PValue = JudyHSGet(JGUID_object_map, (void *)object, (Word_t)object_type?object_type*16+1:strlen(object+1)+2);
-    if (unlikely(!PValue)) {
-        uv_rwlock_rdunlock(&global_lock);
-        return 1;
-    }
-
-    if (likely(uuid))
-        uuid_copy(*uuid, *PValue);
-    uv_rwlock_rdunlock(&global_lock);
-    return 0;
-}
-
-int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate)
-{
-    char  *target_object;
-    uuid_t temp_uuid;
-    int rc;
-
-    switch (object_type) {
-        case GUID_TYPE_DIMENSION:
-            if (unlikely(find_or_generate_guid((void *) ((RRDDIM *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
-                return 1;
-            target_object = mallocz(49);
-            target_object[0] = object_type;
-            memcpy(target_object + 1, ((RRDDIM *)object)->rrdset->rrdhost->host_uuid, 16);
-            memcpy(target_object + 17, ((RRDDIM *)object)->rrdset->chart_uuid, 16);
-            memcpy(target_object + 33, temp_uuid, 16);
-            break;
-        case GUID_TYPE_CHART:
-            if (unlikely(find_or_generate_guid((void *) ((RRDSET *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
-                return 1;
-            target_object = mallocz(33);
-            target_object[0] = object_type;
-            memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
-            memcpy(target_object + 17, temp_uuid, 16);
-            break;
-        case GUID_TYPE_HOST:
-            target_object = mallocz(17);
-            target_object[0] = object_type;
-            memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16);
-            break;
-        case GUID_TYPE_CHAR:
-            target_object = mallocz(strlen((char *) object)+2);
-            target_object[0] = object_type;
-            strcpy(target_object+1, (char *) object);
-            break;
-        default:
-            return 1;
-    }
-    rc = find_guid_by_object(target_object, uuid, object_type);
-    if (rc) {
-        if (!replace_instead_of_generate) /* else take *uuid as user input */
-            uuid_generate(*uuid);
-        uv_rwlock_wrlock(&global_lock);
-        rc = guid_store_nolock(uuid, target_object, object_type);
-        uv_rwlock_wrunlock(&global_lock);
-        if (rc)
-            freez(target_object);
-        return rc;
-    }
-#ifdef NETDATA_INTERNAL_CHECKS
-    dump_object(uuid, target_object);
-#endif
-    freez(target_object);
-    return 0;
-}
-
-void init_global_guid_map()
-{
-    static int init = 0;
-
-    if (init)
-        return;
-
-    init = 1;
-    info("Configuring locking mechanism for global GUID map");
-    fatal_assert(0 == uv_rwlock_init(&guid_lock));
-    fatal_assert(0 == uv_rwlock_init(&object_lock));
-    fatal_assert(0 == uv_rwlock_init(&global_lock));
-    return;
-}
-
-
diff --git a/database/engine/global_uuid_map/global_uuid_map.h b/database/engine/global_uuid_map/global_uuid_map.h
deleted file mode 100644
index f31f3c0079..0000000000
--- a/database/engine/global_uuid_map/global_uuid_map.h
+++ /dev/null
@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_GLOBAL_UUID_MAP_H
-#define NETDATA_GLOBAL_UUID_MAP_H
-
-#include "libnetdata/libnetdata.h"
-#include <Judy.h>
-#include "../../rrd.h"
-
-typedef enum guid_type {
-    GUID_TYPE_CHAR,
-    GUID_TYPE_HOST,
-    GUID_TYPE_CHART,
-    GUID_TYPE_DIMENSION,
-    GUID_TYPE_NOTFOUND,
-    GUID_TYPE_NOSPACE
-} GUID_TYPE;
-
-extern GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes);
-extern int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE);
-extern void init_global_guid_map();
-extern int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate);
-extern void free_uuid(uuid_t *uuid);
-extern void free_global_guid_map();
-#endif //NETDATA_GLOBAL_UUID_MAP_H
diff --git a/database/engine/metadata_log/compaction.c b/database/engine/metadata_log/compaction.c
index c261cd9454..ba19e1edfb 100644
--- a/database/engine/metadata_log/compaction.c
+++ b/database/engine/metadata_log/compaction.c
@@ -3,319 +3,6 @@
 
 #include "metadatalog.h"
 
-void after_compact_old_records(struct metalog_worker_config* wc)
-{
-    struct metalog_instance *ctx = wc->ctx;
-    int error;
-
-    mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log, &ctx->compaction_state.new_metadata_logfiles);
-    uv_run(wc->loop, UV_RUN_DEFAULT);
-
-    error = uv_thread_join(wc->now_compacting_files);
-    if (error) {
-        error("uv_thread_join(): %s", uv_strerror(error));
-    }
-    freez(wc->now_compacting_files);
-    /* unfreeze command processing */
-    wc->now_compacting_files = NULL;
-
-    wc->cleanup_thread_compacting_files = 0;
-
-    /* interrupt event loop */
-    uv_stop(wc->loop);
-
-    info("Finished metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
-}
-
-static void metalog_flush_compaction_records(struct metalog_instance *ctx)
-{
-    struct metalog_cmd cmd;
-    struct completion compaction_completion;
-
-    init_completion(&compaction_completion);
-
-    cmd.opcode = METALOG_COMPACTION_FLUSH;
-    cmd.record_io_descr.completion = &compaction_completion;
-    metalog_enq_cmd(&ctx->worker_config, &cmd);
-
-    wait_for_completion(&compaction_completion);
-    destroy_completion(&compaction_completion);
-}
-
-/* The caller must have called metalog_flush_compaction_records() before to synchronize and quiesce the event loop. */
-static void compaction_test_quota(struct metalog_worker_config *wc)
-{
-    struct metalog_instance *ctx = wc->ctx;
-    struct logfile_compaction_state *compaction_state;
-    struct metadata_logfile *oldmetalogfile, *newmetalogfile;
-    unsigned current_size;
-    int ret;
-
-    compaction_state = &ctx->compaction_state;
-    newmetalogfile = compaction_state->new_metadata_logfiles.last;
-
-    oldmetalogfile = ctx->metadata_logfiles.first;
-
-    current_size = newmetalogfile->pos;
-    if (unlikely(current_size >= MAX_METALOGFILE_SIZE && newmetalogfile->starting_fileno < oldmetalogfile->fileno)) {
-        /* It's safe to finalize the compacted metadata log file and create a new one since it has already replaced
-         * an older one. */
-
-        /* Finalize as the immediately previous file than the currently compacted one. */
-        ret = rename_metadata_logfile(newmetalogfile, 0, newmetalogfile->fileno - 1);
-        if (ret < 0)
-            return;
-
-        ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles,
-                                       ctx->metadata_logfiles.first->fileno, ctx->metadata_logfiles.first->fileno);
-
-        if (likely(!ret)) {
-            compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
-        }
-    }
-}
-
-
-static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
-{
-    GUID_TYPE ret;
-    RRDSET *st;
-    RRDDIM *rd;
-    BUFFER *buffer;
-    RRDHOST *host = NULL;
-
-    ret = find_object_by_guid(uuid, NULL, 0);
-    switch (ret) {
-        case GUID_TYPE_CHAR:
-            error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR");
-            break;
-        case GUID_TYPE_CHART:
-            st = metalog_get_chart_from_uuid(ctx, uuid);
-            if (st) {
-                if (ctx->current_compaction_id > st->rrdhost->compaction_id) {
-                    error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id);
-                    compact_record_by_uuid(ctx, &st->rrdhost->host_uuid);
-                }
-
-                if (ctx->current_compaction_id > st->compaction_id) {
-                    st->compaction_id = ctx->current_compaction_id;
-                    buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
-                    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
-                } else {
-                    debug(D_METADATALOG, "Chart has already been compacted, ignoring record.");
-                }
-            } else {
-                debug(D_METADATALOG, "Ignoring nonexistent chart metadata record.");
-            }
-            break;
-        case GUID_TYPE_DIMENSION:
-            rd = metalog_get_dimension_from_uuid(ctx, uuid);
-            if (rd) {
-                if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) {
-                    error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname);
-                    compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid);
-                }
-                if (ctx->current_compaction_id > rd->rrdset->compaction_id) {
-                    error("Forcing compaction of CHART %s", rd->rrdset->id);
-                    compact_record_by_uuid(ctx, rd->rrdset->chart_uuid);
-                } else if (ctx->current_compaction_id > rd->state->compaction_id) {
-                    rd->state->compaction_id = ctx->current_compaction_id;
-                    buffer = metalog_update_dimension_buffer(rd);
-                    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
-                } else {
-                    debug(D_METADATALOG, "Dimension has already been compacted, ignoring record.");
-                }
-            } else {
-                debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record.");
-            }
-            break;
-        case GUID_TYPE_HOST:
-            host = metalog_get_host_from_uuid(ctx, uuid);
-            if (unlikely(!host))
-                break;
-            if (ctx->current_compaction_id > host->compaction_id) {
-                host->compaction_id = ctx->current_compaction_id;
-                buffer = metalog_update_host_buffer(host);
-                metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
-            } else {
-                debug(D_METADATALOG, "Host has already been compacted, ignoring record.");
-            }
-            break;
-        case GUID_TYPE_NOTFOUND:
-            debug(D_METADATALOG, "Ignoring nonexistent metadata record.");
-            break;
-        case GUID_TYPE_NOSPACE:
-            error_with_guid(uuid, "Not enough space for object retrieval");
-            break;
-        default:
-            error("Unknown return code %u from find_object_by_guid", ret);
-            break;
-    }
-}
-
-/* Returns 0 on success. */
-static int compact_metadata_logfile_records(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
-{
-    struct metalog_worker_config* wc = &ctx->worker_config;
-    struct logfile_compaction_state *compaction_state;
-    struct metalog_record *record;
-    struct metalog_record_block *record_block, *prev_record_block;
-    int ret;
-    unsigned iterated_records;
-#define METADATA_LOG_RECORD_BATCH 128 /* Flush I/O and check sizes whenever this many records have been iterated */
-
-    info("Compacting metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-         ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
-
-    compaction_state = &ctx->compaction_state;
-    record_block = prev_record_block = NULL;
-    iterated_records = 0;
-    for (record = mlf_record_get_first(metalogfile) ; record != NULL ; record = mlf_record_get_next(metalogfile)) {
-        if ((record_block = metalogfile->records.iterator.current) != prev_record_block) {
-            if (prev_record_block) { /* Deallocate iterated record blocks */
-                rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
-                freez(prev_record_block);
-            }
-            prev_record_block = record_block;
-        }
-        compact_record_by_uuid(ctx, &record->uuid);
-        if (0 == ++iterated_records % METADATA_LOG_RECORD_BATCH) {
-            metalog_flush_compaction_records(ctx);
-            if (compaction_state->throttle) {
-                (void)sleep_usec(10000); /* 10 msec throttle compaction */
-            }
-            compaction_test_quota(wc);
-        }
-    }
-    if (prev_record_block) { /* Deallocate iterated record blocks */
-        rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
-        freez(prev_record_block);
-    }
-
-    info("Compacted metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-         ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
-
-    metadata_logfile_list_delete(&ctx->metadata_logfiles, metalogfile);
-    ret = destroy_metadata_logfile(metalogfile);
-    if (!ret) {
-        info("Deleted file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-             ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
-        rrd_atomic_fetch_add(&ctx->disk_space, -metalogfile->pos);
-    } else {
-        error("Failed to delete file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-             ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
-    }
-    freez(metalogfile);
-
-    return ret;
-}
-
-static void compact_old_records(void *arg)
-{
-    struct metalog_instance *ctx = arg;
-    struct metalog_worker_config* wc = &ctx->worker_config;
-    struct logfile_compaction_state *compaction_state;
-    struct metadata_logfile *metalogfile, *nextmetalogfile, *newmetalogfile;
-    int ret;
-
-    compaction_state = &ctx->compaction_state;
-
-    nextmetalogfile = NULL;
-    for (metalogfile = ctx->metadata_logfiles.first ;
-         metalogfile != compaction_state->last_original_logfile ;
-         metalogfile = nextmetalogfile) {
-        nextmetalogfile = metalogfile->next;
-
-        newmetalogfile = compaction_state->new_metadata_logfiles.last;
-        ret = rename_metadata_logfile(newmetalogfile, newmetalogfile->starting_fileno, metalogfile->fileno);
-        if (ret < 0) {
-            error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-                  ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
-        }
-
-        ret = compact_metadata_logfile_records(ctx, metalogfile);
-        if (ret) {
-            error("Metadata log compaction failed, cancelling.");
-            break;
-        }
-    }
-    fatal_assert(nextmetalogfile); /* There are always more than 1 metadata log files during compaction */
-
-    newmetalogfile = compaction_state->new_metadata_logfiles.last;
-    if (newmetalogfile->starting_fileno != 0) { /* Must rename the last compacted file */
-        ret = rename_metadata_logfile(newmetalogfile, 0, nextmetalogfile->fileno - 1);
-        if (ret < 0) {
-            error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
-                  ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
-        }
-    }
-    /* Connect the compacted files to the metadata log */
-    newmetalogfile->next = nextmetalogfile;
-    ctx->metadata_logfiles.first = compaction_state->new_metadata_logfiles.first;
-
-    wc->cleanup_thread_compacting_files = 1;
-    /* wake up event loop */
-    fatal_assert(0 == uv_async_send(&wc->async));
-}
-
-/* Returns 0 on success. */
-static int init_compaction_state(struct metalog_instance *ctx)
-{
-    struct metadata_logfile *newmetalogfile;
-    struct logfile_compaction_state *compaction_state;
-    int ret;
-
-    compaction_state = &ctx->compaction_state;
-    compaction_state->new_metadata_logfiles.first = NULL;
-    compaction_state->new_metadata_logfiles.last = NULL;
-    compaction_state->starting_fileno = ctx->metadata_logfiles.first->fileno;
-    compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
-    compaction_state->last_original_logfile = ctx->metadata_logfiles.last;
-    compaction_state->throttle = 0;
-
-    ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, compaction_state->starting_fileno,
-                                   compaction_state->fileno);
-    if (unlikely(ret)) {
-        error("Cannot create new metadata log files, compaction aborted.");
-        return ret;
-    }
-    newmetalogfile = compaction_state->new_metadata_logfiles.first;
-    fatal_assert(newmetalogfile == compaction_state->new_metadata_logfiles.last);
-    init_metadata_record_log(&compaction_state->records_log);
-
-    return 0;
-}
-
-void metalog_do_compaction(struct metalog_worker_config *wc)
-{
-    struct metalog_instance *ctx = wc->ctx;
-    int error;
-
-    if (wc->now_compacting_files) {
-        /* already compacting metadata log files */
-        return;
-    }
-    wc->now_compacting_files = mallocz(sizeof(*wc->now_compacting_files));
-    wc->cleanup_thread_compacting_files = 0;
-    metalog_try_link_new_metadata_logfile(wc);
-
-    error = init_compaction_state(ctx);
-    if (unlikely(error)) {
-        error("Cannot create new metadata log files, compaction aborted.");
-        return;
-    }
-    ++ctx->current_compaction_id; /* Signify a new compaction */
-
-    info("Starting metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
-    error = uv_thread_create(wc->now_compacting_files, compact_old_records, ctx);
-    if (error) {
-        error("uv_thread_create(): %s", uv_strerror(error));
-        freez(wc->now_compacting_files);
-        wc->now_compacting_files = NULL;
-    }
-
-}
-
 /* Return 0 on success. */
 int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
                                  unsigned *matched_files)
diff --git a/database/engine/metadata_log/compaction.h b/database/engine/metadata_log/compaction.h
index da4765eeb5..d046134403 100644
--- a/database/engine/metadata_log/compaction.h
+++ b/database/engine/metadata_log/compaction.h
@@ -8,19 +8,7 @@
 #endif
 #include "../rrdengine.h"
 
-struct logfile_compaction_state {
-    unsigned fileno; /* Starts at 1 */
-    unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
-
-    struct metadata_record_commit_log records_log;
-    struct metadata_logfile_list new_metadata_logfiles;
-    struct metadata_logfile *last_original_logfile; /* Marks the end of compaction */
-    uint8_t throttle; /* set non-zero to throttle compaction */
-};
-
 extern int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
                                        unsigned *matched_files);
-extern void metalog_do_compaction(struct metalog_worker_config *wc);
-extern void after_compact_old_records(struct metalog_worker_config* wc);
 
 #endif /* NETDATA_COMPACTION_H */
diff --git a/database/engine/metadata_log/logfile.c b/database/engine/metadata_log/logfile.c
index 08bb4eec86..b7c5c06182 100644
--- a/database/engine/metadata_log/logfile.c
+++ b/database/engine/metadata_log/logfile.c
@@ -1,184 +1,8 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
+#include <database/sqlite/sqlite_functions.h>
 #include "metadatalog.h"
 #include "metalogpluginsd.h"
 
-static void mlf_record_block_insert(struct metadata_logfile *metalogfile, struct metalog_record_block *record_block)
-{
-
-    if (likely(NULL != metalogfile->records.last)) {
-        metalogfile->records.last->next = record_block;
-    }
-    if (unlikely(NULL == metalogfile->records.first)) {
-        metalogfile->records.first = record_block;
-    }
-    metalogfile->records.last = record_block;
-}
-
-void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record)
-{
-    struct metalog_record_block *record_block;
-    struct metalog_instance *ctx = metalogfile->ctx;
-
-    record_block = metalogfile->records.last;
-    if (likely(NULL != record_block && record_block->records_nr < MAX_METALOG_RECORDS_PER_BLOCK)) {
-        record_block->record_array[record_block->records_nr++] = *record;
-    } else { /* Create n