Reorganize and cleanup database related code (#17101)

* Rearrange code

* Explicit init of sqlite library and shutdown
Function to close all databases

* Add functions to return db size

* Prevent overflow

* Address review comments

* Initialize sqlite library before ml_init

* Fix unittest

13 files changed, 799 insertions, 770 deletions

diff --git a/src/daemon/main.c b/src/daemon/main.c
index 46f008cb56..d1e795f231 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -448,11 +448,10 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 #endif
     }
 
-    sql_close_context_database();
-    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
+    sqlite_close_databases();
+    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);
+    sqlite_library_shutdown();
 
-    sql_close_database();
-    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
 
     // unlink the pid
     if(pidfile[0]) {
@@ -1500,22 +1499,24 @@ int main(int argc, char **argv) {
 #endif
 
                         if(strcmp(optarg, "sqlite-meta-recover") == 0) {
-                            sql_init_database(DB_CHECK_RECOVER, 0);
+                            sql_init_meta_database(DB_CHECK_RECOVER, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "sqlite-compact") == 0) {
-                            sql_init_database(DB_CHECK_RECLAIM_SPACE, 0);
+                            sql_init_meta_database(DB_CHECK_RECLAIM_SPACE, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "sqlite-analyze") == 0) {
-                            sql_init_database(DB_CHECK_ANALYZE, 0);
+                            sql_init_meta_database(DB_CHECK_ANALYZE, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "unittest") == 0) {
                             unittest_running = true;
+                            if (sqlite_library_init())
+                                return 1;
 
                             if (pluginsd_parser_unittest()) return 1;
                             if (unit_test_static_threads()) return 1;
@@ -1539,6 +1540,7 @@ int main(int argc, char **argv) {
                             if (ctx_unittest()) return 1;
                             if (uuid_unittest()) return 1;
                             if (dyncfg_unittest()) return 1;
+                            sqlite_library_shutdown();
                             fprintf(stderr, "\n\nALL TESTS PASSED\n\n");
                             return 0;
                         }
@@ -2054,6 +2056,8 @@ int main(int argc, char **argv) {
                 exit(1);
             }
         }
+        if (sqlite_library_init())
+            fatal("Failed to initialize sqlite library");
 
         // --------------------------------------------------------------------
         // Initialize ML configuration
diff --git a/src/daemon/unit_test.c b/src/daemon/unit_test.c
index 1b350fecd3..183770baf4 100644
--- a/src/daemon/unit_test.c
+++ b/src/daemon/unit_test.c
@@ -2546,7 +2546,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
 
     fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");
 
-    (void) sql_init_database(DB_CHECK_NONE, 1);
+    (void)sql_init_meta_database(DB_CHECK_NONE, 1);
     host = dbengine_rrdhost_find_or_create("dbengine-stress-test");
     if (NULL == host)
         return;
diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c
index e19d9e2d6f..3eea22019d 100644
--- a/src/daemon/watcher.c
+++ b/src/daemon/watcher.c
@@ -85,8 +85,7 @@ void *watcher_main(void *arg)
     watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
     watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
     watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
-    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
-    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
+    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);
     watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE);
     watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);
     watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);
@@ -146,10 +145,8 @@ void watcher_thread_start() {
         "wait for dbengine main cache to finish flushing";
     watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg =
         "stop dbengine tiers";
-    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg =
-        "close SQL context db";
-    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg =
-        "close SQL main db";
+    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_DATABASES].msg =
+        "close SQL databases";
     watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg =
         "remove pid file";
     watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg =
diff --git a/src/daemon/watcher.h b/src/daemon/watcher.h
index 6c9ce15523..b785ca4362 100644
--- a/src/daemon/watcher.h
+++ b/src/daemon/watcher.h
@@ -27,8 +27,7 @@ typedef enum {
     WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH,
     WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING,
     WATCHER_STEP_ID_STOP_DBENGINE_TIERS,
-    WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB,
-    WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB,
+    WATCHER_STEP_ID_CLOSE_SQL_DATABASES,
     WATCHER_STEP_ID_REMOVE_PID_FILE,
     WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES,
     WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE,
diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c
index bb12013423..9bf4853a38 100644
--- a/src/database/rrdhost.c
+++ b/src/database/rrdhost.c
@@ -1009,7 +1009,7 @@ void dbengine_init(char *hostname) {
 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
     rrdhost_init();
 
-    if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
+    if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
         if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
             set_late_global_environment(system_info);
             fatal("Failed to initialize SQLite");
diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c
index 683ed12cd3..ad76a1ee2c 100644
--- a/src/database/sqlite/sqlite_context.c
+++ b/src/database/sqlite/sqlite_context.c
@@ -71,24 +71,6 @@ int sql_init_context_database(int memory)
     return 0;
 }
 
-/*
- * Close the sqlite database
- */
-
-void sql_close_context_database(void)
-{
-    int rc;
-    if (unlikely(!db_context_meta))
-        return;
-
-    netdata_log_info("Closing context SQLite database");
-
-    rc = sqlite3_close_v2(db_context_meta);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));
-    db_context_meta = NULL;
-}
-
 //
 // Fetching data
 //
@@ -422,6 +404,12 @@ int sql_context_cache_stats(int op)
     return count;
 }
 
+
+uint64_t sqlite_get_context_space(void)
+{
+    return sqlite_get_db_space(db_context_meta);
+}
+
 //
 // TESTING FUNCTIONS
 //
@@ -456,7 +444,8 @@ int ctx_unittest(void)
     uuid_t host_uuid;
     uuid_generate(host_uuid);
 
-    initialize_thread_key_pool();
+    if (sqlite_library_init())
+        return 1;
 
     int rc = sql_init_context_database(1);
 
@@ -532,7 +521,8 @@ int ctx_unittest(void)
     ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL);
     netdata_log_info("List context end after delete");
 
-    sql_close_context_database();
+    sql_close_database(db_context_meta, "CONTEXT");
+    sqlite_library_shutdown();
 
     return 0;
 }
diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h
index 2586916eae..92d02fdd22 100644
--- a/src/database/sqlite/sqlite_context.h
+++ b/src/database/sqlite/sqlite_context.h
@@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);
 int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data);
 
 int sql_init_context_database(int memory);
+uint64_t sqlite_get_context_space(void);
 void sql_close_context_database(void);
 int ctx_unittest(void);
 #endif //NETDATA_SQLITE_CONTEXT_H
diff --git a/src/database/sqlite/sqlite_functions.c b/src/database/sqlite/sqlite_functions.c
index 2b01ce79c6..1dc2022b37 100644
--- a/src/database/sqlite/sqlite_functions.c
+++ b/src/database/sqlite/sqlite_functions.c
@@ -1,94 +1,6 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "sqlite_functions.h"
-#include "sqlite3recover.h"
-#include "sqlite_db_migration.h"
-
-#define DB_METADATA_VERSION 16
-
-const char *database_config[] = {
-    "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
-    "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "
-    "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default '',"
-    "hops INT NOT NULL DEFAULT 0,"
-    "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"
-    "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "
-    "entries INT NOT NULL DEFAULT 0,"
-    "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)",
-
-    "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "
-    "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "
-    "chart_type int, memory_mode int, history_entries)",
-
-    "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "
-    "multiplier int, divisor int , algorithm int, options text)",
-
-    "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)",
-
-    "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)",
-
-    "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)",
-
-    "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "
-    "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))",
-
-    "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)",
-
-    "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "
-    "on_key text, class text, component text, type text, os text, hosts text, lookup text, "
-    "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "
-    "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
-    "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
-    "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text)",
-
-    "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "
-    "date_created INT, PRIMARY KEY(host_id, system_key))",
-
-    "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "
-    "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))",
-
-    "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)"
-      " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END",
-
-    "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, "
-    "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "
-    "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))",
-
-    "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)",
-
-    "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "
-    "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "
-    "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "
-    "info text, exec_code int, new_status real, old_status real, delay int, "
-    "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)",
-
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",
-
-    NULL
-};
-
-const char *database_cleanup[] = {
-    "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)",
-    "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)",
-    "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)",
-    "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)",
-    "DROP TRIGGER IF EXISTS tr_dim_del",
-    "DROP INDEX IF EXISTS ind_d1",
-    "DROP INDEX IF EXISTS ind_c1",
-    "DROP INDEX IF EXISTS ind_c2",
-    "DROP INDEX IF EXISTS alert_hash_index",
-    "DROP INDEX IF EXISTS health_log_d_ind_4",
-    "DROP INDEX IF EXISTS health_log_d_ind_1",
-    "DROP INDEX IF EXISTS health_log_d_ind_5",
-    NULL
-};
-
-sqlite3 *db_meta = NULL;
 
 #define MAX_PREPARED_STATEMENTS (32)
 pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];
@@ -152,45 +64,6 @@ static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database)
     return false;
 }
 
-static void recover_database(const char *sqlite_database, const char *new_sqlite_database)
-{
-    sqlite3 *database;
-    int rc = sqlite3_open(sqlite_database, &database);
-    if (rc != SQLITE_OK)
-        return;
-
-    netdata_log_info("Recover %s", sqlite_database);
-    netdata_log_info("     to %s", new_sqlite_database);
-
-    // This will remove the -shm and -wal files when we close the database
-    (void) db_execute(database, "select count(*) from sqlite_master limit 0");
-
-    sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);
-    if (recover) {
-
-        rc = sqlite3_recover_run(recover);
-
-        if (rc == SQLITE_OK)
-            netdata_log_info("Recover complete");
-        else
-            netdata_log_info("Recover encountered an error but the database may be usable");
-
-        rc = sqlite3_recover_finish(recover);
-
-        (void) sqlite3_close(database);
-
-        if (rc == SQLITE_OK) {
-            rc = rename(new_sqlite_database, sqlite_database);
-            if (rc == 0) {
-                netdata_log_info("Renamed %s", new_sqlite_database);
-                netdata_log_info("     to %s", sqlite_database);
-            }
-        }
-    }
-    else
-        (void) sqlite3_close(database);
-}
-
 int execute_insert(sqlite3_stmt *res)
 {
     int rc;
@@ -279,14 +152,11 @@ static void add_stmt_to_list(sqlite3_stmt *res)
 static void release_statement(void *statement)
 {
     int rc;
-#ifdef NETDATA_DEV_MODE
-    netdata_log_info("Thread %d: Cleaning prepared statement on %p", gettid(), statement);
-#endif
     if (unlikely(rc = sqlite3_finalize((sqlite3_stmt *) statement) != SQLITE_OK))
         error_report("Failed to finalize statement, rc = %d", rc);
 }
 
-void initialize_thread_key_pool(void)
+static void initialize_thread_key_pool(void)
 {
     for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++)
         (void)pthread_key_create(&key_pool[i], release_statement);
@@ -303,13 +173,9 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state
         key = &key_pool[keys_used++];
 
     int rc = sqlite3_prepare_v2(database, query, -1, statement, 0);
-    if (likely(rc == SQLITE_OK)) {
-        if (likely(key)) {
+    if (rc == SQLITE_OK) {
+        if (key)
             ret = pthread_setspecific(*key, *statement);
-#ifdef NETDATA_DEV_MODE
-            netdata_log_info("Thread %d: Using key %u on statement %p", gettid(), keys_used, *statement);
-#endif
-        }
         if (ret)
             add_stmt_to_list(*statement);
     }
@@ -355,202 +221,6 @@ int init_database_batch(sqlite3 *database, const char *batch[], const char *desc
     return 0;
 }
 
-static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)
-{
-    uuid_t  uuid;
-
-    if ( argc != 1 ){
-        sqlite3_result_null(context);
-        return ;
-    }
-    int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);
-    if (rc == -1)  {
-        sqlite3_result_null(context);
-        return ;
-    }
-
-    sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);
-}
-
-void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv)
-{
-    if (argc != 1 ){
-        sqlite3_result_null(context);
-        return ;
-    }
-
-    if (sqlite3_value_int(argv[0]) != 0) {
-        struct timespec req = {.tv_sec = 0, .tv_nsec = 1};
-        nanosleep(&req, NULL);
-    }
-
-    sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec());
-}
-
-void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv)
-{
-    (void)argc;
-    (void)argv;
-
-    uuid_t uuid;
-    uuid_generate_random(uuid);
-    sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);
-}
-
-/*
- * Initialize the SQLite database
- * Return 0 on success
- */
-int sql_init_database(db_check_action_type_t rebuild, int memory)
-{
-    char *err_msg = NULL;
-    char sqlite_database[FILENAME_MAX + 1];
-    int rc;
-
-    if (likely(!memory)) {
-        snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
-        rc = unlink(sqlite_database);
-        snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
-
-        if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {
-            char new_sqlite_database[FILENAME_MAX + 1];
-            snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);
-            recover_database(sqlite_database, new_sqlite_database);
-            if (rebuild & DB_CHECK_RECOVER)
-                return 0;
-        }
-    }
-    else
-        strcpy(sqlite_database, ":memory:");
-
-    rc = sqlite3_open(sqlite_database, &db_meta);
-    if (rc != SQLITE_OK) {
-        error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
-        char *error_str = get_database_extented_error(db_meta, 0, "meta_open");
-        if (error_str)
-            analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);
-        freez(error_str);
-        sqlite3_close(db_meta);
-        db_meta = NULL;
-        return 1;
-    }
-
-    if (rebuild & DB_CHECK_RECLAIM_SPACE) {
-        netdata_log_info("Reclaiming space of %s", sqlite_database);
-        rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg);
-        if (rc != SQLITE_OK) {
-            error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);
-            sqlite3_free(err_msg);
-        }
-        else {
-            (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");
-            (void) sqlite3_close(db_meta);
-        }
-        return 1;
-    }
-
-    if (rebuild & DB_CHECK_ANALYZE) {
-        errno = 0;
-        netdata_log_info("Running ANALYZE on %s", sqlite_database);
-        rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);
-        if (rc != SQLITE_OK) {
-            error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg);
-            sqlite3_free(err_msg);
-        }
-        else {
-            (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");
-            (void) sqlite3_close(db_meta);
-        }
-        return 1;
-    }
-
-    netdata_log_info("SQLite database %s initialization", sqlite_database);
-
-    rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to register internal u2h function");
-
-    rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to register internal now_usec function");
-
-    rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to register internal uuid_random function");
-
-    int target_version = DB_METADATA_VERSION;
-
-    if (likely(!memory))
-        target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
-
-   if (configure_sqlite_database(db_meta, target_version, "meta_config"))
-        return 1;
-
-    if (init_database_batch(db_meta, &database_config[0], "meta_init"))
-        return 1;
-
-    if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup"))
-        return 1;
-
-    netdata_log_info("SQLite database initialization completed");
-
-    initialize_thread_key_pool();
-
-    return 0;
-}
-
-/*
- * Close the sqlite database
- */
-
-void sql_close_database(void)
-{
-    int rc;
-    if (unlikely(!db_meta))
-        return;
-
-    netdata_log_info("Closing SQLite database");
-
-    add_stmt_to_list(NULL);
-
-    (void) db_execute(db_meta, "PRAGMA analysis_limit=10000");
-    (void) db_execute(db_meta, "PRAGMA optimize");
-
-    rc = sqlite3_close_v2(db_meta);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
-    db_meta = NULL;
-}
-
-int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
-{
-    int rc, result = 1;
-    sqlite3_stmt *res = NULL;
-
-    rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement %s, rc = %d", sql, rc);
-        return 1;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);
-        goto skip;
-    }
-
-    rc = execute_insert(res);
-    if (likely(rc == SQLITE_DONE))
-        result = SQLITE_OK;
-    else
-        error_report("Failed to execute %s, rc = %d", sql, rc);
-
-skip:
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize statement %s, rc = %d", sql, rc);
-    return result;
-}
 // Return 0 OK
 // Return 1 Failed
 int db_execute(sqlite3 *db, const char *cmd)
@@ -580,376 +250,124 @@ int db_execute(sqlite3 *db, const char *cmd)
     return (rc != SQLITE_OK);
 }
 
-static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
+// Utils
+int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null)
 {
-    if (unlikely(!host))
-        return;
-
-    if (unlikely(!node_id)) {
-        freez(host->node_id);
-        __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED);
-        return;
-    }
-
-    struct aclk_sync_cfg_t  *wc = host->aclk_config;
-
-    if (unlikely(!host->node_id)) {
-        uuid_t *t = mallocz(sizeof(*host->node_id));
-        uuid_copy(*t, *node_id);
-        __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED);
-    }
-    else {
-        uuid_copy(*(host->node_id), *node_id);
-    }
-
-    if (unlikely(!wc))
-        sql_create_aclk_table(host, &host->host_uuid, node_id);
-    else
-        uuid_unparse_lower(*node_id, wc->node_id);
+    if (likely(text))
+        return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC);
+    if (!can_be_null)
+        return 1;
+    return sqlite3_bind_null(res, position);
 }
 
-#define SQL_UPDATE_NODE_ID  "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id"
+#define SQL_DROP_TABLE "DROP table %s"
 
-int update_node_id(uuid_t *host_id, uuid_t *node_id)
+void sql_drop_table(const char *table)
 {
-    sqlite3_stmt *res = NULL;
-    RRDHOST *host = NULL;
-    int rc = 2;
-
-    char host_guid[GUID_LEN + 1];
-    uuid_unparse_lower(*host_id, host_guid);
-    rrd_wrlock();
-    host = rrdhost_find_by_guid(host_guid);
-    if (likely(host))
-        set_host_node_id(host, node_id);
-    rrd_unlock();
-
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return 1;
-    }
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to store node instance information");
-        return 1;
-    }
+    if (!table)
+        return;
 
-    rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id parameter to store node instance information");
-        goto failed;
-    }
+    char wstr[255];
+    snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table);
 
-    rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id parameter to store node instance information");
-        goto failed;
+    int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL);
+    if (rc != SQLITE_OK) {
+        error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc);
     }
-
-    rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE))
-        error_report("Failed to store node instance information, rc = %d", rc);
-    rc = sqlite3_changes(db_meta);
-
-failed:
-    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when storing node instance information");
-
-    return rc - 1;
 }
 
-#define SQL_SELECT_NODE_ID  "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL"
-
-int get_node_id(uuid_t *host_id, uuid_t *node_id)
+static int get_pragma_value(sqlite3 *database, const char *sql)
 {
     sqlite3_stmt *res = NULL;
-    int rc;
-
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return 1;
-    }
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to select node instance information for a host");
-        return 1;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id parameter to select node instance information");
-        goto failed;
-    }
+    int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK))
+        return -1;
 
+    int result = -1;
     rc = sqlite3_step_monitored(res);
-    if (likely(rc == SQLITE_ROW && node_id))
-        uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
+    if (likely(rc == SQLITE_ROW))
+        result = sqlite3_column_int(res, 0);
 
-failed:
-    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when selecting node instance information");
+    rc = sqlite3_finalize(res);
+    (void) rc;
 
-    return (rc == SQLITE_ROW) ? 0 : -1;
+    return result;
 }
 
-#define SQL_INVALIDATE_NODE_INSTANCES                                                                                  \
-    "UPDATE node_instance SET node_id = NULL WHERE EXISTS "                                                            \
-    "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))"
-
-void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)
+int get_free_page_count(sqlite3 *database)
 {
-    sqlite3_stmt *res = NULL;
-    int rc;
-
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return;
-    }
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to invalidate node instance ids");
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id parameter to invalidate node instance information");
-        goto failed;
-    }
-
-    if (claim_id)
-        rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);
-    else
-        rc = sqlite3_bind_null(res, 2);
-
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind claim_id parameter to invalidate node instance information");
-        goto failed;
-    }
-
-    rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE))
-        error_report("Failed to invalidate node instance information, rc = %d", rc);
-
-failed:
-    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when invalidating node instance information");
+    return get_pragma_value(database, "PRAGMA freelist_count");
 }
 
-#define SQL_GET_NODE_INSTANCE_LIST                                                                                     \
-    "SELECT ni.node_id, ni.host_id, h.hostname "                                                                       \
-    "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0"
-
-struct node_instance_list *get_node_list(void)
+int get_database_page_count(sqlite3 *database)
 {
-    struct node_instance_list *node_list = NULL;
-    sqlite3_stmt *res = NULL;
-    int rc;
-
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return NULL;
-    }
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to get node instance information");
-        return NULL;
-    }
-
-    int row = 0;
-    char host_guid[37];
-    while (sqlite3_step_monitored(res) == SQLITE_ROW)
-        row++;
-
-    if (sqlite3_reset(res) != SQLITE_OK) {
-        error_report("Failed to reset the prepared statement while fetching node instance information");
-        goto failed;
-    }
-    node_list = callocz(row + 1, sizeof(*node_list));
-    int max_rows = row;
-    row = 0;
-    // TODO: Check to remove lock
-    rrd_rdlock();
-    while (sqlite3_step_monitored(res) == SQLITE_ROW) {
-        if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))
-            uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
-        if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
-            uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
-            uuid_unparse_lower(*host_id, host_guid);
-            RRDHOST *host = rrdhost_find_by_guid(host_guid);
-            if (!host)
-                continue;
-            if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {
-                netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));
-                continue;
-            }
-            uuid_copy(node_list[row].host_id, *host_id);
-            node_list[row].queryable = 1;
-            node_list[row].live = (host && (host == localhost || host->receiver
-                                            || !(rrdhost_flag_check(host, RRDHO