Use sqlite to store the health log and alert configurations. (#11399)

* Rebased

* use sql health log if it exists

* store alert config in sqlite

* move unlock before loop

* fix warnings

* remove hash message

* check return from counting health log

* remove check of hostname when reading log

* try to create the health log table to catch accidental removals of it

* fix warnings, cast values, report config_hash_id

* use snprintfz, add info logging

* remove unnecessary strdup and free

* check if stored config hash is null

* return if prepare statement fails

* replace with static variables

* remove replace info, free edit_command

* remove setting cfg entries to NULL

* change uuid_copy

* check return of uuid_parse, and exit if its not valid

* also free cfg

* use address

* removed health_alarm_entry_sql2json and sql_health_alarm_log_select_all

* remove check for is_valid_alarm_id

* replace lengths with GUID_LEN

* use uuid_unparse_lower_fix

* removed web api endopoint to get alert config

* check for non null values for name, chart and family

* include a date_updated field in alert_hash

* for config hash, digest NULL string if value to digest is null

* Use empty string instead of null

5 files changed, 172 insertions, 33 deletions

diff --git a/health/health.c b/health/health.c
index ccd2603176..5782e54f0f 100644
--- a/health/health.c
+++ b/health/health.c
@@ -662,6 +662,8 @@ void *health_main(void *ptr) {
     int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
     if(min_run_every < 1) min_run_every = 1;
 
+    int cleanup_sql_every_loop = 7200 / min_run_every;
+
     time_t now                = now_realtime_sec();
     time_t hibernation_delay  = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
 
@@ -716,6 +718,9 @@ void *health_main(void *ptr) {
                 host->health_delay_up_to = 0;
             }
 
+            if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
+                sql_health_alarm_log_cleanup(host);
+
             rrdhost_rdlock(host);
 
             // the first loop is to lookup values from the db
@@ -956,7 +961,7 @@ void *health_main(void *ptr) {
 
                         if(likely(!rrdcalc_isrepeating(rc))) {
                             ALARM_ENTRY *ae = health_create_alarm_entry(
-                                    host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,
+                                    host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
                                     rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
                                     rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,
                                     rc->delay_last,
@@ -1006,7 +1011,7 @@ void *health_main(void *ptr) {
                     if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
                         rc->last_repeat = now;
                         ALARM_ENTRY *ae = health_create_alarm_entry(
-                                host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id,
+                                host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
                                 rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
                                 rc->old_value, rc->value, rc->old_status, rc->status, rc->source, rc->units, rc->info,
                                 rc->delay_last,
diff --git a/health/health.h b/health/health.h
index e7408f632d..00629e9821 100644
--- a/health/health.h
+++ b/health/health.h
@@ -63,6 +63,7 @@ extern ALARM_ENTRY* health_create_alarm_entry(
         RRDHOST *host,
         uint32_t alarm_id,
         uint32_t alarm_event_id,
+        uuid_t config_hash_id,
         time_t when,
         const char *name,
         const char *chart,
diff --git a/health/health_config.c b/health/health_config.c
index a79f727aae..64a7346433 100644
--- a/health/health_config.c
+++ b/health/health_config.c
@@ -503,6 +503,40 @@ static inline void strip_quotes(char *s) {
     }
 }
 
+static inline void alert_config_free(struct alert_config *cfg)
+{
+    freez(cfg->alarm);
+    freez(cfg->template_key);
+    freez(cfg->os);
+    freez(cfg->host);
+    freez(cfg->on);
+    freez(cfg->families);
+    freez(cfg->plugin);
+    freez(cfg->module);
+    freez(cfg->charts);
+    freez(cfg->lookup);
+    freez(cfg->calc);
+    freez(cfg->warn);
+    freez(cfg->crit);
+    freez(cfg->every);
+    freez(cfg->green);
+    freez(cfg->red);
+    freez(cfg->exec);
+    freez(cfg->to);
+    freez(cfg->units);
+    freez(cfg->info);
+    freez(cfg->classification);
+    freez(cfg->component);
+    freez(cfg->type);
+    freez(cfg->delay);
+    freez(cfg->options);
+    freez(cfg->repeat);
+    freez(cfg->host_labels);
+    freez(cfg->p_db_lookup_dimensions);
+    freez(cfg->p_db_lookup_method);
+    freez(cfg);
+}
+
 static int health_readfile(const char *filename, void *data) {
     RRDHOST *host = (RRDHOST *)data;
 
@@ -577,6 +611,7 @@ static int health_readfile(const char *filename, void *data) {
 
     RRDCALC *rc = NULL;
     RRDCALCTEMPLATE *rt = NULL;
+    struct alert_config *alert_cfg = NULL;
 
     int ignore_this = 0;
     size_t line = 0, append = 0;
@@ -626,16 +661,18 @@ static int health_readfile(const char *filename, void *data) {
 
         if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
             if(rc) {
-                if(ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) {
+                if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) {
                     rrdcalc_free(rc);
+                    alert_config_free(alert_cfg);
                 }
                // health_add_alarms_loop(host, rc, ignore_this) ;
             }
 
             if(rt) {
-                if (ignore_this || !rrdcalctemplate_add_template_from_config(host, rt))
+                if (ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) {
                     rrdcalctemplate_free(rt);
-
+                    alert_config_free(alert_cfg);
+                }
                 rt = NULL;
             }
 
@@ -652,25 +689,30 @@ static int health_readfile(const char *filename, void *data) {
             rc->old_status = RRDCALC_STATUS_UNINITIALIZED;
             rc->warn_repeat_every = host->health_default_warn_repeat_every;
             rc->crit_repeat_every = host->health_default_crit_repeat_every;
+            alert_cfg = callocz(1, sizeof(struct alert_config));
 
             if(rrdvar_fix_name(rc->name))
                 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
 
+            alert_cfg->alarm = strdupz(rc->name);
             ignore_this = 0;
         }
         else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
             if(rc) {
 //                health_add_alarms_loop(host, rc, ignore_this) ;
-                if(ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) {
+                if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) {
                     rrdcalc_free(rc);
+                    alert_config_free(alert_cfg);
                 }
 
                 rc = NULL;
             }
 
             if(rt) {
-                if(ignore_this || !rrdcalctemplate_add_template_from_config(host, rt))
+                if(ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) {
                     rrdcalctemplate_free(rt);
+                    alert_config_free(alert_cfg);
+                }
             }
 
             rt = callocz(1, sizeof(RRDCALCTEMPLATE));
@@ -682,14 +724,17 @@ static int health_readfile(const char *filename, void *data) {
             rt->delay_multiplier = 1.0;
             rt->warn_repeat_every = host->health_default_warn_repeat_every;
             rt->crit_repeat_every = host->health_default_crit_repeat_every;
+            alert_cfg = callocz(1, sizeof(struct alert_config));
 
             if(rrdvar_fix_name(rt->name))
                 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
 
+            alert_cfg->template_key = strdupz(rt->name);
             ignore_this = 0;
         }
         else if(hash == hash_os && !strcasecmp(key, HEALTH_OS_KEY)) {
             char *os_match = value;
+            alert_cfg->os = strdupz(value);
             SIMPLE_PATTERN *os_pattern = simple_pattern_create(os_match, NULL, SIMPLE_PATTERN_EXACT);
 
             if(!simple_pattern_matches(os_pattern, host->os)) {
@@ -706,6 +751,7 @@ static int health_readfile(const char *filename, void *data) {
         }
         else if(hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
             char *host_match = value;
+            alert_cfg->host = strdupz(value);
             SIMPLE_PATTERN *host_pattern = simple_pattern_create(host_match, NULL, SIMPLE_PATTERN_EXACT);
 
             if(!simple_pattern_matches(host_pattern, host->hostname)) {
@@ -722,6 +768,7 @@ static int health_readfile(const char *filename, void *data) {
         }
         else if(rc) {
             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
+                alert_cfg->on = strdupz(value);
                 if(rc->chart) {
                     if(strcmp(rc->chart, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -733,6 +780,7 @@ static int health_readfile(const char *filename, void *data) {
                 rc->hash_chart = simple_hash(rc->chart);
             }
             else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) {
+                alert_cfg->classification = strdupz(value);
                 if(rc->classification) {
                     if(strcmp(rc->classification, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -744,6 +792,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rc->classification);
             }
             else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) {
+                alert_cfg->component = strdupz(value);
                 if(rc->component) {
                     if(strcmp(rc->component, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -755,6 +804,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rc->component);
             }
             else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) {
+                alert_cfg->type = strdupz(value);
                 if(rc->type) {
                     if(strcmp(rc->type, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -766,18 +816,32 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rc->type);
             }
             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
+                alert_cfg->lookup = strdupz(value);
                 health_parse_db_lookup(line, filename, value, &rc->group, &rc->after, &rc->before,
                         &rc->update_every, &rc->options, &rc->dimensions, &rc->foreachdim);
                 if(rc->foreachdim) {
                     rc->spdim = health_pattern_from_foreach(rc->foreachdim);
                 }
+                if (rc->after) {
+                    if (rc->dimensions)
+                        alert_cfg->p_db_lookup_dimensions = strdupz(rc->dimensions);
+                    if (rc->group)
+                        alert_cfg->p_db_lookup_method = strdupz(group_method2string(rc->group));
+                    alert_cfg->p_db_lookup_options = rc->options;
+                    alert_cfg->p_db_lookup_after = rc->after;
+                    alert_cfg->p_db_lookup_before = rc->before;
+                    alert_cfg->p_update_every = rc->update_every;
+                }
             }
             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
+                alert_cfg->every = strdupz(value);
                 if(!config_parse_duration(value, &rc->update_every))
                     error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
                             line, filename, rc->name, key, value);
+                alert_cfg->p_update_every = rc->update_every;
             }
             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
+                alert_cfg->green = strdupz(value);
                 char *e;
                 rc->green = str2ld(value, &e);
                 if(e && *e) {
@@ -786,6 +850,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
+                alert_cfg->red = strdupz(value);
                 char *e;
                 rc->red = str2ld(value, &e);
                 if(e && *e) {
@@ -794,6 +859,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
+                alert_cfg->calc = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rc->calculation = expression_parse(value, &failed_at, &error);
@@ -803,6 +869,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
+                alert_cfg->warn = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rc->warning = expression_parse(value, &failed_at, &error);
@@ -812,6 +879,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
+                alert_cfg->crit = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rc->critical = expression_parse(value, &failed_at, &error);
@@ -821,6 +889,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
+                alert_cfg->exec = strdupz(value);
                 if(rc->exec) {
                     if(strcmp(rc->exec, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -831,6 +900,7 @@ static int health_readfile(const char *filename, void *data) {
                 rc->exec = strdupz(value);
             }
             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
+                alert_cfg->to = strdupz(value);
                 if(rc->recipient) {
                     if(strcmp(rc->recipient, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -841,6 +911,7 @@ static int health_readfile(const char *filename, void *data) {
                 rc->recipient = strdupz(value);
             }
             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
+                alert_cfg->units = strdupz(value);
                 if(rc->units) {
                     if(strcmp(rc->units, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -852,6 +923,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rc->units);
             }
             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
+                alert_cfg->info = strdupz(value);
                 if(rc->info) {
                     if(strcmp(rc->info, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -863,17 +935,21 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rc->info);
             }
             else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
+                alert_cfg->delay = strdupz(value);
                 health_parse_delay(line, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
             }
             else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
+                alert_cfg->options = strdupz(value);
                 rc->options |= health_parse_options(value);
             }
             else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){
+                alert_cfg->repeat = strdupz(value);
                 health_parse_repeat(line, filename, value,
                                     &rc->warn_repeat_every,
                                     &rc->crit_repeat_every);
             }
             else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
+                alert_cfg->host_labels = strdupz(value);
                 if(rc->labels) {
                     if(strcmp(rc->labels, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
@@ -887,6 +963,7 @@ static int health_readfile(const char *filename, void *data) {
                 rc->splabels = simple_pattern_create(rc->labels, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
+                alert_cfg->plugin = strdupz(value);
                 freez(rc->plugin_match);
                 simple_pattern_free(rc->plugin_pattern);
 
@@ -894,6 +971,7 @@ static int health_readfile(const char *filename, void *data) {
                 rc->plugin_pattern = simple_pattern_create(rc->plugin_match, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) {
+                alert_cfg->module = strdupz(value);
                 freez(rc->module_match);
                 simple_pattern_free(rc->module_pattern);
 
@@ -907,6 +985,7 @@ static int health_readfile(const char *filename, void *data) {
         }
         else if(rt) {
             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
+                alert_cfg->on = strdupz(value);
                 if(rt->context) {
                     if(strcmp(rt->context, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -918,6 +997,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->hash_context = simple_hash(rt->context);
             }
             else if(hash == hash_class && !strcasecmp(key, HEALTH_CLASS_KEY)) {
+                alert_cfg->classification = strdupz(value);
                 if(rt->classification) {
                     if(strcmp(rt->classification, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -929,6 +1009,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rt->classification);
             }
             else if(hash == hash_component && !strcasecmp(key, HEALTH_COMPONENT_KEY)) {
+                alert_cfg->component = strdupz(value);
                 if(rt->component) {
                     if(strcmp(rt->component, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -940,6 +1021,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rt->component);
             }
             else if(hash == hash_type && !strcasecmp(key, HEALTH_TYPE_KEY)) {
+                alert_cfg->type = strdupz(value);
                 if(rt->type) {
                     if(strcmp(rt->type, value) != 0)
                         error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -951,6 +1033,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rt->type);
             }
             else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
+                alert_cfg->families = strdupz(value);
                 freez(rt->family_match);
                 simple_pattern_free(rt->family_pattern);
 
@@ -958,6 +1041,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->family_pattern = simple_pattern_create(rt->family_match, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
+                alert_cfg->plugin = strdupz(value);
                 freez(rt->plugin_match);
                 simple_pattern_free(rt->plugin_pattern);
 
@@ -965,6 +1049,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->plugin_pattern = simple_pattern_create(rt->plugin_match, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_module && !strcasecmp(key, HEALTH_MODULE_KEY)) {
+                alert_cfg->module = strdupz(value);
                 freez(rt->module_match);
                 simple_pattern_free(rt->module_pattern);
 
@@ -972,6 +1057,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->module_pattern = simple_pattern_create(rt->module_match, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_charts && !strcasecmp(key, HEALTH_CHARTS_KEY)) {
+                alert_cfg->charts = strdupz(value);
                 freez(rt->charts_match);
                 simple_pattern_free(rt->charts_pattern);
 
@@ -979,18 +1065,32 @@ static int health_readfile(const char *filename, void *data) {
                 rt->charts_pattern = simple_pattern_create(rt->charts_match, NULL, SIMPLE_PATTERN_EXACT);
             }
             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
+                alert_cfg->lookup = strdupz(value);
                 health_parse_db_lookup(line, filename, value, &rt->group, &rt->after, &rt->before,
                         &rt->update_every, &rt->options, &rt->dimensions, &rt->foreachdim);
                 if(rt->foreachdim) {
                     rt->spdim = health_pattern_from_foreach(rt->foreachdim);
                 }
+                if (rt->after) {
+                    if (rt->dimensions)
+                        alert_cfg->p_db_lookup_dimensions = strdupz(rt->dimensions);
+                    if (rt->group)
+                        alert_cfg->p_db_lookup_method = strdupz(group_method2string(rt->group));
+                    alert_cfg->p_db_lookup_options = rt->options;
+                    alert_cfg->p_db_lookup_after = rt->after;
+                    alert_cfg->p_db_lookup_before = rt->before;
+                    alert_cfg->p_update_every = rt->update_every;
+                }
             }
             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
+                alert_cfg->every = strdupz(value);
                 if(!config_parse_duration(value, &rt->update_every))
                     error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
                             line, filename, rt->name, key, value);
+                alert_cfg->p_update_every = rt->update_every;
             }
             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
+                alert_cfg->green = strdupz(value);
                 char *e;
                 rt->green = str2ld(value, &e);
                 if(e && *e) {
@@ -999,6 +1099,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
+                alert_cfg->red = strdupz(value);
                 char *e;
                 rt->red = str2ld(value, &e);
                 if(e && *e) {
@@ -1007,6 +1108,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
+                alert_cfg->calc = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rt->calculation = expression_parse(value, &failed_at, &error);
@@ -1016,6 +1118,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
+                alert_cfg->warn = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rt->warning = expression_parse(value, &failed_at, &error);
@@ -1025,6 +1128,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
+                alert_cfg->crit = strdupz(value);
                 const char *failed_at = NULL;
                 int error = 0;
                 rt->critical = expression_parse(value, &failed_at, &error);
@@ -1034,6 +1138,7 @@ static int health_readfile(const char *filename, void *data) {
                 }
             }
             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
+                alert_cfg->exec = strdupz(value);
                 if(rt->exec) {
                     if(strcmp(rt->exec, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -1044,6 +1149,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->exec = strdupz(value);
             }
             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
+                alert_cfg->to = strdupz(value);
                 if(rt->recipient) {
                     if(strcmp(rt->recipient, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -1054,6 +1160,7 @@ static int health_readfile(const char *filename, void *data) {
                 rt->recipient = strdupz(value);
             }
             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
+                alert_cfg->units = strdupz(value);
                 if(rt->units) {
                     if(strcmp(rt->units, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -1065,6 +1172,7 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rt->units);
             }
             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
+                alert_cfg->info = strdupz(value);
                 if(rt->info) {
                     if(strcmp(rt->info, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -1076,17 +1184,21 @@ static int health_readfile(const char *filename, void *data) {
                 strip_quotes(rt->info);
             }
             else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
+                alert_cfg->delay = strdupz(value);
                 health_parse_delay(line, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
             }
             else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
+                alert_cfg->options = strdupz(value);
                 rt->options |= health_parse_options(value);
             }
             else if(hash == hash_repeat && !strcasecmp(key, HEALTH_REPEAT_KEY)){
+                alert_cfg->repeat = strdupz(value);
                 health_parse_repeat(line, filename, value,
                                     &rt->warn_repeat_every,
                                     &rt->crit_repeat_every);
             }
             else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
+                alert_cfg->host_labels = strdupz(value);
                 if(rt->labels) {
                     if(strcmp(rt->labels, value) != 0)
                         error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
@@ -1112,14 +1224,17 @@ static int health_readfile(const char *filename, void *data) {
 
     if(rc) {
         //health_add_alarms_loop(host, rc, ignore_this) ;
-        if(ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) {
+        if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) {
             rrdcalc_free(rc);
+            alert_config_free(alert_cfg);
         }
     }
 
     if(rt) {
-        if(ignore_this || !rrdcalctemplate_add_template_from_config(host, rt))
+        if(ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) {
             rrdcalctemplate_free(rt);
+            alert_config_free(alert_cfg);
+        }
     }
 
     fclose(fp);
diff --git a/health/health_json.c b/health/health_json.c
index d3c3073215..a21d5a4fd6 100644
--- a/health/health_json.c
+++ b/health/health_json.c
@@ -15,6 +15,8 @@ void health_string2json(BUFFER *wb, const char *prefix, const char *label, const
 
 void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
     char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0");
+    char config_hash_id[GUID_LEN + 1];
+    uuid_unparse_lower(ae->config_hash_id, config_hash_id);
 
     buffer_sprintf(wb,
             "\n\t{\n"
@@ -24,6 +26,7 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
                     "\t\t\"unique_id\": %u,\n"
                     "\t\t\"alarm_id\": %u,\n"
                     "\t\t\"alarm_event_id\": %u,\n"
+                    "\t\t\"config_hash_id\": \"%s\",\n"
                     "\t\t\"name\": \"%s\",\n"
                     "\t\t\"chart\": \"%s\",\n"
                     "\t\t\"family\": \"%s\",\n"
@@ -59,6 +62,7 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
                    , ae->unique_id
                    , ae->alarm_id
                    , ae->alarm_event_id
+                   , config_hash_id
                    , ae->name
                    , ae->chart
                    , ae->family
@@ -187,9 +191,13 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
         }
     }
 
+    char hash_id[GUID_LEN + 1];
+    uuid_unparse_lower(rc->config_hash_id, hash_id);
+
     buffer_sprintf(wb,
             "\t\t\"%s.%s\": {\n"
                     "\t\t\t\"id\": %lu,\n"
+                    "\t\t\t\"config_hash_id\": \"%s\",\n"
                     "\t\t\t\"name\": \"%s\",\n"
                     "\t\t\t\"chart\": \"%s\",\n"
                     "\t\t\t\"family\": \"%s\",\n"
@@ -221,6 +229,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
                     "\t\t\t\"last_repeat\": \"%lu\",\n"
                    , rc->chart, rc->name
                    , (unsigned long)rc->id
+                   , hash_id
                    , rc->name
                    , rc->chart
                    , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
diff --git a/health/health_log.c b/health/health_log.c
index bb9416fd2f..e5d0e0fa96 100644
--- a/health/health_log.c
+++ b/health/health_log.c
@@ -38,39 +38,41 @@ static inline void health_log_rotate(RRDHOST *host) {
     }
 
     if(unlikely(host->health_log_entries_written > rotate_every)) {
-        health_alarm_log_close(host);
+        if(unlikely(host->health_log_fp)) {
+            health_alarm_log_close(host);
 
-        char old_filename[FILENAME_MAX + 1];
-        snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
+            char old_filename[FILENAME_MAX + 1];
+            snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
 
-        if(unlink(old_filename) == -1 && errno != ENOENT)
-            error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, old_filename);
+            if(unlink(old_filename) == -1 && errno != ENOENT)
+                error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, old_filename);
 
-        if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
-            error("HEALTH [%s]: cannot move file '%s' to '%s'.", host->hostname, host->health_log_filename, old_filename);
+            if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
+                error("HEALTH [%s]: cannot move file '%s' to '%s'.", host->hostname, host->health_log_filename, old_filename);
 
-        if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
-            error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, host->health_log_filename);
+            if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
+                error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, host->health_log_filename);
 
-        // open it with truncate
-        host->health_log_fp = fopen(host->health_log_filename, "w");
+            // open it with truncate
+            host->health_log_fp = fopen(host->health_log_filename, "w");
 
-        if(host->health_log_fp)
-            fclose(host->health_log_fp);
-        else
-            error("HEALTH [%s]: cannot truncate health log '%s'", host->hostname, host->health_log_filename);
+            if(host->health_log_fp)
+                fclose(host->health_log_fp);
+            else
+                error("HEALTH [%s]: cannot truncate health log '%s'", host->hostname, host->health_log_filename);
 
-        host->health_log_fp = NULL;
+            host->health_log_fp = NULL;
 
-        host->health_log_entries_written = 0;
-        health_alarm_log_open(host);
+            host->health_log_entries_written = 0;
+            health_alarm_log_open(host);
+        }
     }
 }
 
 inline void health_label_log_save(RRDHOST *host) {
     health_log_rotate(host);
 
-    if(likely(host->health_log_fp)) {
+    if(unlikely(host->health_log_fp)) {
         BUFFER *wb = buffer_create(1024);
         rrdhost_check_rdlock(host);
         netdata_rwlock_rdlock(&host->labels.labels_rwlock);
@@ -101,7 +103,7 @@ inline void health_label_log_save(RRDHOST *host) {
 
 inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
     health_log_rotate(host);