Use sqlite to store the health log and alert configurations. (#11399)

* Rebased

* use sql health log if it exists

* store alert config in sqlite

* move unlock before loop

* fix warnings

* remove hash message

* check return from counting health log

* remove check of hostname when reading log

* try to create the health log table to catch accidental removals of it

* fix warnings, cast values, report config_hash_id

* use snprintfz, add info logging

* remove unnecessary strdup and free

* check if stored config hash is null

* return if prepare statement fails

* replace with static variables

* remove replace info, free edit_command

* remove setting cfg entries to NULL

* change uuid_copy

* check return of uuid_parse, and exit if its not valid

* also free cfg

* use address

* removed health_alarm_entry_sql2json and sql_health_alarm_log_select_all

* remove check for is_valid_alarm_id

* replace lengths with GUID_LEN

* use uuid_unparse_lower_fix

* removed web api endopoint to get alert config

* check for non null values for name, chart and family

* include a date_updated field in alert_hash

* for config hash, digest NULL string if value to digest is null

* Use empty string instead of null

9 files changed, 1034 insertions, 7 deletions

diff --git a/database/rrd.h b/database/rrd.h
index a5163e9b9f..ba6ef4d0b2 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -35,6 +35,7 @@ struct pg_cache_page_index;
 #include "rrdcalctemplate.h"
 #include "streaming/rrdpush.h"
 #include "aclk/aclk_rrdhost_state.h"
+#include "sqlite/sqlite_health.h"
 
 enum {
     CONTEXT_FLAGS_ARCHIVE = 0x01,
@@ -650,6 +651,7 @@ struct alarm_entry {
     uint32_t unique_id;
     uint32_t alarm_id;
     uint32_t alarm_event_id;
+    uuid_t config_hash_id;
 
     time_t when;
     time_t duration;
diff --git a/database/rrdcalc.c b/database/rrdcalc.c
index 85b9efb75c..1b1a149603 100644
--- a/database/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -87,6 +87,7 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
                 host,
                 rc->id,
                 rc->next_event_id++,
+                rc->config_hash_id,
                 now,
                 rc->name,
                 rc->rrdset->id,
@@ -164,6 +165,7 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
                 host,
                 rc->id,
                 rc->next_event_id++,
+                rc->config_hash_id,
                 now,
                 rc->name,
                 rc->rrdset->id,
@@ -398,6 +400,7 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
     rc->hash = simple_hash(rc->name);
     rc->chart = strdupz(chart);
     rc->hash_chart = simple_hash(rc->chart);
+    uuid_copy(rc->config_hash_id, rt->config_hash_id);
 
     rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);
 
@@ -513,6 +516,7 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch
     newrc->hash = simple_hash(newrc->name);
     newrc->chart = strdupz(rc->chart);
     newrc->hash_chart = simple_hash(rc->chart);
+    uuid_copy(newrc->config_hash_id, *((uuid_t *) &rc->config_hash_id));
 
     newrc->dimensions = strdupz(dimension);
     newrc->foreachdim = NULL;
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
index b4122c6056..d7446f63ad 100644
--- a/database/rrdcalc.h
+++ b/database/rrdcalc.h
@@ -38,6 +38,7 @@ struct rrdcalc {
 
     char *name;                     // the name of this alarm
     uint32_t hash;                  // the hash of the alarm name
+    uuid_t config_hash_id;          // a predictable hash_id based on specific alert configuration
 
     char *exec;                     // the command to execute when this alarm switches state
     char *recipient;                // the recipient of the alarm (the first parameter to exec)
@@ -149,6 +150,43 @@ struct rrdcalc {
     struct rrdcalc *next;
 };
 
+struct alert_config {
+    char *alarm;
+    char *template_key;
+    char *os;
+    char *host;
+    char *on;
+    char *families;
+    char *plugin;
+    char *module;
+    char *charts;
+    char *lookup;
+    char *calc;
+    char *warn;
+    char *crit;
+    char *every;
+    char *green;
+    char *red;
+    char *exec;
+    char *to;
+    char *units;
+    char *info;
+    char *classification;
+    char *component;
+    char *type;
+    char *delay;
+    char *options;
+    char *repeat;
+    char *host_labels;
+
+    char *p_db_lookup_dimensions;
+    char *p_db_lookup_method;
+    uint32_t p_db_lookup_options;
+    int32_t p_db_lookup_after;
+    int32_t p_db_lookup_before;
+    int32_t p_update_every;
+};
+
 extern int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id);
 extern int alarm_entry_isrepeating(RRDHOST *host, ALARM_ENTRY *ae);
 extern RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name, uint32_t hash);
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 65114da6af..0f12bba059 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -11,6 +11,7 @@
 struct rrdcalctemplate {
     char *name;
     uint32_t hash_name;
+    uuid_t config_hash_id;
 
     char *exec;
     char *recipient;
diff --git a/database/rrdhost.c b/database/rrdhost.c
index dfd6a7af31..7922822b87 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -295,9 +295,6 @@ RRDHOST *rrdhost_create(const char *hostname,
         rrdhost_wrlock(host);
         health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
         rrdhost_unlock(host);
-
-        health_alarm_log_load(host);
-        health_alarm_log_open(host);
     }
 
     RRDHOST *t = rrdhost_index_add(host);
@@ -313,6 +310,23 @@ RRDHOST *rrdhost_create(const char *hostname,
         if (unlikely(rc))
             error_report("Failed to store machine GUID to the database");
         sql_load_node_id(host);
+        if (host->health_enabled) {
+            if (!file_is_migrated(host->health_log_filename)) {
+                int rc = sql_create_health_log_table(host);
+                if (unlikely(rc)) {
+                    error_report("Failed to create health log table in the database");
+                    health_alarm_log_load(host);
+                    health_alarm_log_open(host);
+                }
+                else {
+                    health_alarm_log_load(host);
+                    add_migrated_file(host->health_log_filename, 0);
+                }
+            } else {
+                sql_create_health_log_table(host);
+                sql_health_alarm_log_load(host);
+            }
+        }
     }
     else
         error_report("Host machine GUID %s is not valid", host->machine_guid);
@@ -506,8 +520,21 @@ void rrdhost_update(RRDHOST *host
             health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
             rrdhost_unlock(host);
 
-            health_alarm_log_load(host);
-            health_alarm_log_open(host);
+            if (!file_is_migrated(host->health_log_filename)) {
+                int rc = sql_create_health_log_table(host);
+                if (unlikely(rc)) {
+                    error_report("Failed to create health log table in the database");
+
+                    health_alarm_log_load(host);
+                    health_alarm_log_open(host);
+                } else {
+                    health_alarm_log_load(host);
+                    add_migrated_file(host->health_log_filename, 0);
+                }
+            } else {
+                sql_create_health_log_table(host);
+                sql_health_alarm_log_load(host);
+            }
         }
         rrd_hosts_available++;
         info("Host %s is not in archived mode anymore", host->hostname);
diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c
index 0258401120..51c6bb691c 100644
--- a/database/sqlite/sqlite_functions.c
+++ b/database/sqlite/sqlite_functions.c
@@ -20,6 +20,12 @@ const char *database_config[] = {
     "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "
     "label_value text, date_created int, PRIMARY KEY (chart_id, label_key));",
     "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created);",
+    "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "
+    "on_key text, class text, component text, type text, os text, hosts text, lookup text, "
+    "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "
+    "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
+    "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
+    "p_db_lookup_after int, p_db_lookup_before int, p_update_every int);",
     "delete from chart_active;",
     "delete from dimension_active;",
     "delete from chart where chart_id not in (select chart_id from dimension);",
@@ -32,7 +38,7 @@ sqlite3 *db_meta = NULL;
 
 static uv_mutex_t sqlite_transaction_lock;
 
-static int execute_insert(sqlite3_stmt *res)
+int execute_insert(sqlite3_stmt *res)
 {
     int rc;
 
@@ -66,7 +72,7 @@ static void add_stmt_to_list(sqlite3_stmt *res)
     statements[idx++] = res;
 }
 
-static int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement) {
+int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement) {
     int rc = sqlite3_prepare_v2(database, query, -1, statement, 0);
     if (likely(rc == SQLITE_OK))
         add_stmt_to_list(*statement);
diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h
index 373316fec9..4573a3c58d 100644
--- a/database/sqlite/sqlite_functions.h
+++ b/database/sqlite/sqlite_functions.h
@@ -78,4 +78,6 @@ extern int get_host_id(uuid_t *node_id, uuid_t *host_id);
 extern void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
 extern struct node_instance_list *get_node_list(void);
 extern void sql_load_node_id(RRDHOST *host);
+extern int execute_insert(sqlite3_stmt *res);
+extern int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
 #endif //NETDATA_SQLITE_FUNCTIONS_H
diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c
new file mode 100644
index 0000000000..c34e2c902a
--- /dev/null
+++ b/database/sqlite/sqlite_health.c
@@ -0,0 +1,931 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sqlite_health.h"
+#include "sqlite_functions.h"
+
+#define MAX_HEALTH_SQL_SIZE 2048
+
+/* Health related SQL queries
+   Creates a health log table in sqlite, one per host guid
+*/
+#define SQL_CREATE_HEALTH_LOG_TABLE(guid) "CREATE TABLE IF NOT EXISTS health_log_%s(hostname text, unique_id int, alarm_id int, alarm_event_id int, config_hash_id blob, updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, flags int, exec_run_timestamp int, delay_up_to_timestamp int, name text, chart text, family text, exec text, recipient text, source text, units text, info text, exec_code int, new_status real, old_status real, delay int, new_value double, old_value double, last_repeat int, class text, component text, type text);", guid
+int sql_create_health_log_table(RRDHOST *host) {
+    int rc;
+    char *err_msg = NULL, command[MAX_HEALTH_SQL_SIZE + 1];
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
+        return 1;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CREATE_HEALTH_LOG_TABLE(uuid_str));
+
+    rc = sqlite3_exec(db_meta, command, 0, 0, &err_msg);
+    if (rc != SQLITE_OK) {
+        error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", host->hostname, rc, err_msg);
+        sqlite3_free(err_msg);
+        return 1;
+    }
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, "CREATE INDEX IF NOT EXISTS "
+            "health_log_index_%s ON health_log_%s (unique_id); ", uuid_str, uuid_str);
+    db_execute(command);
+
+    return 0;
+}
+
+/* Health related SQL queries
+   Updates an entry in the table
+*/
+#define SQL_UPDATE_HEALTH_LOG(guid) "UPDATE health_log_%s set updated_by_id = ?, flags = ?, exec_run_timestamp = ?, exec_code = ? where unique_id = ?;", guid
+void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
+    sqlite3_stmt *res = NULL;
+    int rc;
+    char command[MAX_HEALTH_SQL_SIZE + 1];
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
+        return;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_UPDATE_HEALTH_LOG(uuid_str));
+
+    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("HEALTH [%s]: Failed to prepare statement for SQL_UPDATE_HEALTH_LOG", host->hostname);
+        return;
+    }
+
+    rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) ae->updated_by_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind updated_by_id parameter for SQL_UPDATE_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) ae->flags);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind flags parameter for SQL_UPDATE_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) ae->exec_run_timestamp);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind exec_run_timestamp parameter for SQL_UPDATE_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int(res, 4, ae->exec_code);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind exec_code parameter for SQL_UPDATE_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 5, (sqlite3_int64) ae->unique_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind unique_id parameter for SQL_UPDATE_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = execute_insert(res);
+    if (unlikely(rc != SQLITE_DONE)) {
+        error_report("HEALTH [%s]: Failed to update health log, rc = %d", host->hostname, rc);
+    }
+
+    failed:
+    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+        error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", host->hostname);
+
+    return;
+}
+
+/* Health related SQL queries
+   Inserts an entry in the table
+*/
+#define SQL_INSERT_HEALTH_LOG(guid) "INSERT INTO health_log_%s(hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", guid
+void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
+    sqlite3_stmt *res = NULL;
+    int rc;
+    char command[MAX_HEALTH_SQL_SIZE + 1];
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
+        return;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_INSERT_HEALTH_LOG(uuid_str));
+
+    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("HEALTH [%s]: Failed to prepare statement for SQL_INSERT_HEALTH_LOG", host->hostname);
+        return;
+    }
+
+    rc = sqlite3_bind_text(res, 1, host->hostname, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind hostname parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) ae->unique_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind unique_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) ae->alarm_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind alarm_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 4, (sqlite3_int64) ae->alarm_event_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind alarm_event_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_blob(res, 5, &ae->config_hash_id, sizeof(ae->config_hash_id), SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind config_hash_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 6, (sqlite3_int64) ae->updated_by_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind updated_by_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 7, (sqlite3_int64) ae->updates_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind updates_id parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 8, (sqlite3_int64) ae->when);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind when parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 9, (sqlite3_int64) ae->duration);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind duration parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 10, (sqlite3_int64) ae->non_clear_duration);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind non_clear_duration parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 11, (sqlite3_int64) ae->flags);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind flags parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 12, (sqlite3_int64) ae->exec_run_timestamp);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind exec_run_timestamp parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 13, (sqlite3_int64) ae->delay_up_to_timestamp);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind delay_up_to_timestamp parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 14, ae->name, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind name parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 15, ae->chart, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind chart parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 16, ae->family, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind family parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 17, ae->exec, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind exec parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 18, ae->recipient, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind recipient parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 19, ae->source, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind source parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 20, ae->units, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind host_id parameter to store node instance information");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 21, ae->info, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind info parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int(res, 22, ae->exec_code);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind exec_code parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int(res, 23, ae->new_status);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind new_status parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int(res, 24, ae->old_status);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind old_status parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int(res, 25, ae->delay);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind delay parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_double(res, 26, ae->new_value);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind new_value parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_double(res, 27, ae->old_value);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind old_value parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_int64(res, 28, (sqlite3_int64) ae->last_repeat);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind last_repeat parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 29, ae->classification, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind classification parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 30, ae->component, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind component parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = sqlite3_bind_text(res, 31, ae->type, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind type parameter for SQL_INSERT_HEALTH_LOG");
+        goto failed;
+    }
+
+    rc = execute_insert(res);
+    if (unlikely(rc != SQLITE_DONE)) {
+        error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", host->hostname, rc);
+        goto failed;
+    }
+
+    ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
+    host->health_log_entries_written++;
+
+    failed:
+    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+        error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", host->hostname);
+
+    return;
+}
+
+void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)
+{
+    if (ae->flags & HEALTH_ENTRY_FLAG_SAVED)
+        sql_health_alarm_log_update(host, ae);
+    else
+        sql_health_alarm_log_insert(host, ae);
+}
+
+/* Health related SQL queries
+   Cleans up the health_log table.
+*/
+#define SQL_CLEANUP_HEALTH_LOG(guid,guid2,limit) "DELETE from health_log_%s where unique_id in (SELECT unique_id from health_log_%s order by unique_id asc LIMIT %lu);", guid, guid2, limit
+void sql_health_alarm_log_cleanup(RRDHOST *host) {
+    sqlite3_stmt *res = NULL;
+    static size_t rotate_every = 0;
+    int rc;
+    char command[MAX_HEALTH_SQL_SIZE + 1];
+
+    if(unlikely(rotate_every == 0)) {
+        rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000);
+        if(rotate_every < 100) rotate_every = 100;
+    }
+
+    if(likely(host->health_log_entries_written < rotate_every)) {
+        return;
+    }
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("Database has not been initialized");
+        return;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG(uuid_str, uuid_str, host->health_log_entries_written - rotate_every));
+
+    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to prepare statement to cleanup health log table");
+        return;
+    }
+
+    rc = sqlite3_step(res);
+    if (unlikely(rc != SQLITE_DONE))
+        error_report("Failed to cleanup health log table, rc = %d", rc);
+
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement to cleanup health log table");
+
+    host->health_log_entries_written = rotate_every;
+}
+
+/* Health related SQL queries
+   Get a count of rows from health log table
+*/
+#define SQL_COUNT_HEALTH_LOG(guid) "SELECT count(1) FROM health_log_%s;", guid
+void sql_health_alarm_log_count(RRDHOST *host) {
+    sqlite3_stmt *res = NULL;
+    int rc;
+    char command[MAX_HEALTH_SQL_SIZE + 1];
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("Database has not been initialized");
+        return;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_COUNT_HEALTH_LOG(uuid_str));
+
+    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to prepare statement to count health log entries from db");
+        return;
+    }
+
+    rc = sqlite3_step(res);
+    if (likely(rc == SQLITE_ROW))
+        host->health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);
+
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement to count health log entries from db");
+
+    info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", host->hostname, uuid_str, host->health_log_entries_written);
+}
+
+/* Health related SQL queries
+   Load from the health log table
+*/
+#define SQL_LOAD_HEALTH_LOG(guid,limit) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type FROM (SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type FROM health_log_%s order by unique_id desc limit %u) order by unique_id asc;", guid, limit
+void sql_health_alarm_log_load(RRDHOST *host) {
+    sqlite3_stmt *res = NULL;
+    int rc;
+    ssize_t errored = 0, loaded = 0;
+    char command[MAX_HEALTH_SQL_SIZE + 1];
+
+    host->health_log_entries_written = 0;
+
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("HEALTH [%s]: Database has not been initialized", host->hostname);
+        return;
+    }
+
+    char uuid_str[GUID_LEN + 1];
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+
+    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_LOAD_HEALTH_LOG(uuid_str, host->health_log.max));
+
+    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("HEALTH [%s]: Failed to prepare sql statement to load health log.", host->hostname);
+        return;
+    }
+
+    netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
+
+    while (sqlite3_step(res) == SQLITE_ROW) {
+        ALARM_ENTRY *ae = NULL;
+
+        // check that we have valid ids
+        uint32_t unique_id = (uint32_t) sqlite3_column_int64(res, 1);
+        if(!unique_id) {
+            error_report("HEALTH [%s]: Got invalid unique id. Ignoring it.", host->hostname);
+            errored++;
+            continue;
+        }
+
+        uint32_t alarm_id = (uint32_t) sqlite3_column_int64(res, 2);
+        if(!alarm_id) {
+            error_report("HEALTH [%s]: Got invalid alarm id. Ignoring it.", host->hostname);
+            errored++;
+            continue;
+        }
+
+        //need name, chart and family
+        if (sqlite3_column_type(res, 13) == SQLITE_NULL) {
+            error_report("HEALTH [%s]: Got null name field. Ignoring it.", host->hostname);
+            errored++;
+            continue;
+        }
+
+        if (sqlite3_column_type(res, 14) == SQLITE_NULL) {
+            error_report("HEALTH [%s]: Got null chart field. Ignoring it.", host->hostname);
+            errored++;
+            continue;
+        }
+
+        if (sqlite3_column_type(res, 15) == SQLITE_NULL) {
+            error_report("HEALTH [%s]: Got null family field. Ignoring it.", host->hostname);
+            errored++;
+            continue;
+        }
+
+        // Check if we got last_repeat field
+        time_t last_repeat = 0;
+        last_repeat = (time_t)sqlite3_column_int64(res, 27);
+
+        RRDCALC *rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14)));
+        if (!rc) {
+            for(rc = host->alarms; rc ; rc = rc->next) {
+                RRDCALC *rdcmp  = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_name, (avl_t *)rc);
+                if(rdcmp != rc) {
+                    error("Cannot insert the alarm index ID using log %s", rc->name);
+                }
+            }
+
+            rc = alarm_max_last_repeat(host, (char *) sqlite3_column_text(res, 14), simple_hash((char *) sqlite3_column_text(res, 14)));
+        }
+
+        if(unlikely(rc)) {
+            if (rrdcalc_isrepeating(rc)) {
+                rc->last_repeat = last_repeat;
+                // We iterate through repeating alarm entries only to
+                // find the latest last_repeat timestamp. Otherwise,
+                // there is no need to keep them in memory.
+                continue;
+            }
+        }
+
+        ae = callocz(1, sizeof(ALARM_ENTRY));
+
+        ae->unique_id = unique_id;
+        ae->alarm_id = alarm_id;
+
+        if (sqlite3_column_type(res, 4) != SQLITE_NULL)
+            uuid_copy(ae->config_hash_id, *((uuid_t *) sqlite3_column_blob(res, 4)));
+
+        ae->alarm_event_id = (uint32_t) sqlite3_column_int64(res, 3);
+        ae->updated_by_id = (uint32_t) sqlite3_column_int64(res, 5);
+        ae->updates_id = (uint32_t) sqlite3_column_int64(res, 6);
+
+        ae->when = (time_t) sqlite3_column_int64(res, 7);
+        ae->duration = (time_t) sqlite3_column_int64(res, 8);
+        ae->non_clear_duration = (time_t) sqlite3_column_int64(res, 9);
+
+        ae->flags = (uint32_t) sqlite3_column_int64(res, 10);
+        ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
+
+        ae->exec_run_timestamp = (time_t) sqlite3_column_int64(res, 11);
+        ae->delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 12);
+
+        ae->name = strdupz((char *) sqlite3_column_text(res, 13));
+        ae->hash_name = simple_hash(ae->name);
+
+        ae->chart = strdupz((char *) sqlite3_column_text(res, 14));
+        ae->hash_chart = simple_hash(ae->chart);
+
+        ae->family = strdupz((char *) sqlite3_column_text(res, 15));
+
+        if (sqlite3_column_type(res, 16) != SQLITE_NULL)
+            ae->exec = strdupz((char *) sqlite3_column_text(res, 16));
+        else
+            ae->exec = NULL;
+
+        if (sqlite3_column_type(res, 17) != SQLITE_NULL)
+            ae->recipient = strdupz((char *) sqlite3_column_text(res, 17));
+        else
+            ae->recipient = NULL;
+
+        if (sqlite3_column_type(res, 18) != SQLITE_NULL)
+            ae->source = strdupz((char *) sqlite3_column_text(res, 18));
+        else
+            ae->source = NULL;
+
+        if (sqlite3_column_type(res, 19) != SQLITE_NULL)
+            ae->units = strdupz((char *) sqlite3_column_text(res, 19));
+        else
+            ae->units = NULL;
+
+        if (sqlite3_column_type(res, 20) != SQLITE_NULL)
+            ae->info = strdupz((char *) sqlite3_column_text(res, 20));
+        else
+            ae->info = NULL;
+
+        ae->exec_code   = (int) sqlite3_column_int(res, 21);
+        ae->new_status  = (RRDCALC_STATUS) sqlite3_column_int(res, 22);
+        ae->old_status  = (RRDCALC_STATUS)sqlite3_column_int(res, 23);
+        ae->delay