Initialize foreach alarms of dimensions in health thread. (#12452)

The previous approach required us to try wr-lock the host
after locking a chart and sleeping on failure. Lock contention
would lead to alarms not being created and the agent to become
unresponsive.

2 files changed, 13 insertions, 33 deletions

diff --git a/database/rrd.h b/database/rrd.h
index e48c646019..289cb55133 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -167,7 +167,9 @@ typedef enum rrddim_flags {
     // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at
     // least rrdset_free_obsolete_time seconds ago.
     RRDDIM_FLAG_ARCHIVED                        = (1 << 3),
-    RRDDIM_FLAG_ACLK                            = (1 << 4)
+    RRDDIM_FLAG_ACLK                            = (1 << 4),
+
+    RRDDIM_FLAG_PENDING_FOREACH_ALARM           = (1 << 5), // set when foreach alarm has not been initialized yet
 } RRDDIM_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
@@ -476,6 +478,8 @@ typedef enum rrdset_flags {
     // least rrdset_free_obsolete_time seconds ago.
     RRDSET_FLAG_ARCHIVED            = 1 << 15,
     RRDSET_FLAG_ACLK                = 1 << 16,
+
+    RRDSET_FLAG_PENDING_FOREACH_ALARMS = 1 << 17, // contains dims with uninitialized foreach alarms
 } RRDSET_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
@@ -634,6 +638,7 @@ typedef enum rrdhost_flags {
     RRDHOST_FLAG_EXPORTING_DONT_SEND      = 1 << 4, // don't send it to external databases
     RRDHOST_FLAG_ARCHIVED               = 1 << 5, // The host is archived, no collected charts yet
     RRDHOST_FLAG_MULTIHOST              = 1 << 6, // Host belongs to localhost/megadb
+    RRDHOST_FLAG_PENDING_FOREACH_ALARMS  = 1 << 7, // contains dims with uninitialized foreach alarms
 } RRDHOST_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
diff --git a/database/rrddim.c b/database/rrddim.c
index d79baf2c20..f0514180df 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -3,36 +3,6 @@
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
 
-static inline void calc_link_to_rrddim(RRDDIM *rd)
-{
-    RRDHOST *host = rd->rrdset->rrdhost;
-    RRDSET  *st = rd->rrdset;
-
-    if (st->state && st->state->is_ar_chart)
-        return;
-
-    if (host->alarms_with_foreach || host->alarms_template_with_foreach) {
-        int count = 0;
-        int hostlocked;
-        for (count = 0; count < 5; count++) {
-            hostlocked = netdata_rwlock_trywrlock(&host->rrdhost_rwlock);
-            if (!hostlocked) {
-                rrdcalc_link_to_rrddim(rd, st, host);
-                rrdhost_unlock(host);
-                break;
-            } else if (hostlocked != EBUSY) {
-                error("Cannot lock host to create an alarm for the dimension.");
-            }
-            sleep_usec(USEC_PER_MS * 200);
-        }
-
-        if (count == 5) {
-            error(
-                "Failed to create an alarm for dimension %s of chart %s 5 times. Skipping alarm.", rd->name, st->name);
-        }
-    }
-}
-
 // ----------------------------------------------------------------------------
 // RRDDIM index
 
@@ -248,7 +218,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
             rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
             rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT);
             rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
-            calc_link_to_rrddim(rd);
+
+            rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
+            rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
+            rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
         }
         if (unlikely(rc)) {
             debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
@@ -457,7 +430,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     if(unlikely(rrddim_index_add(st, rd) != rd))
         error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id);
 
-    calc_link_to_rrddim(rd);
+    rrddim_flag_set(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM);
+    rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
+    rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
 
     ml_new_dimension(rd);