Adjust cloud dimension update frequency (#12284)

* Queue a chart immediately to the cloud

* Do not inform the cloud immediately if a dimension stopped collecting use MAX(obsoletion time, 1.5 * update_every)

* Notify cloud immediately on dimension deletion

* Add debug messages

* Do not schedule an update if we are shutting down

6 files changed, 62 insertions, 10 deletions

diff --git a/database/rrd.h b/database/rrd.h
index 1137983794..ba06164010 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -53,7 +53,6 @@ struct context_param {
     uint8_t flags;
 };
 
-#define RRDSET_MINIMUM_LIVE_COUNT 3
 #define META_CHART_UPDATED 1
 #define META_PLUGIN_UPDATED 2
 #define META_MODULE_UPDATED 4
diff --git a/database/rrddim.c b/database/rrddim.c
index 6829cbb3d4..0d1ecc01e2 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -504,6 +504,10 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
         error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
 
     // free(rd->annotations);
+#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+    if (!netdata_exit)
+        aclk_send_dimension_update(rd);
+#endif
 
     RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
     switch(rrd_memory_mode) {
diff --git a/database/rrdhost.c b/database/rrdhost.c
index b33b749b24..8f87784008 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -1516,6 +1516,11 @@ restart_after_removal:
                             }
                             continue;
                         }
+#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+                        else {
+                            aclk_send_dimension_update(rd);
+                        }
+#endif
                     }
                     last = rd;
                     rd = rd->next;
diff --git a/database/rrdset.c b/database/rrdset.c
index 385d7b4d01..ce860d07cd 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -1400,10 +1400,8 @@ void rrdset_done(RRDSET *st) {
 #ifdef ENABLE_ACLK
     if (likely(!st->state->is_ar_chart)) {
         if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
-            if (st->counter_done >= RRDSET_MINIMUM_LIVE_COUNT && st->dimensions) {
-                if (likely(!queue_chart_to_aclk(st)))
-                    rrdset_flag_set(st, RRDSET_FLAG_ACLK);
-            }
+            if (likely(st->dimensions && !queue_chart_to_aclk(st)))
+                rrdset_flag_set(st, RRDSET_FLAG_ACLK);
         }
     }
 #endif
@@ -1832,7 +1830,8 @@ after_second_database_work:
 #if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
     if (likely(!st->state->is_ar_chart)) {
         if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) {
-            int live = ((mark - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
+            int live = ((mark - rd->last_collected_time.tv_sec) <
+                 MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
             if (unlikely(live != rd->state->aclk_live_status)) {
                 if (likely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
                     if (likely(!queue_dimension_to_aclk(rd))) {
@@ -1943,6 +1942,9 @@ after_second_database_work:
                             delete_dimension_uuid(&rd->state->metric_uuid);
                         } else {
                             /* Do not delete this dimension */
+#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+                            aclk_send_dimension_update(rd);
+#endif
                             last = rd;
                             rd = rd->next;
                             continue;
diff --git a/database/sqlite/sqlite_aclk_chart.c b/database/sqlite/sqlite_aclk_chart.c
index eea48a5674..f4e8aea23a 100644
--- a/database/sqlite/sqlite_aclk_chart.c
+++ b/database/sqlite/sqlite_aclk_chart.c
@@ -3,7 +3,7 @@
 #include "sqlite_functions.h"
 #include "sqlite_aclk_chart.h"
 
-#ifdef ENABLE_NEW_CLOUD_PROTOCOL
+#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
 #include "../../aclk/aclk_charts_api.h"
 #include "../../aclk/aclk.h"
 
@@ -277,7 +277,7 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
         time_t first_t = rd->state->query_ops.oldest_time(rd);
         time_t last_t  = rd->state->query_ops.latest_time(rd);
 
-        int live = ((now - last_t) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
+        int live = ((now - last_t) < MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
 
         rc = aclk_upd_dimension_event(
             wc,
@@ -973,6 +973,47 @@ int queue_dimension_to_aclk(RRDDIM *rd)
     return rc;
 }
 
+void aclk_send_dimension_update(RRDDIM *rd)
+{
+    if (!aclk_use_new_cloud_arch)
+        return;
+
+    char *claim_id = is_agent_claimed();
+    if (unlikely(!claim_id))
+        return;
+
+    time_t first_entry_t = rrddim_first_entry_t(rd);
+    time_t last_entry_t = rrddim_last_entry_t(rd);
+
+    time_t now = now_realtime_sec();
+    int live = ((now - rd->last_collected_time.tv_sec) <
+                MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
+
+    if (!live || rd->state->aclk_live_status != live || !first_entry_t) {
+        (void)aclk_upd_dimension_event(
+            rd->rrdset->rrdhost->dbsync_worker,
+            claim_id,
+            &rd->state->metric_uuid,
+            rd->id,
+            rd->name,
+            rd->rrdset->id,
+            first_entry_t,
+            live ? 0 : last_entry_t);
+
+        if (!first_entry_t)
+            debug(D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld)",
+              rd->rrdset->rrdhost->hostname, rd->rrdset->name, rd->name, live, first_entry_t, last_entry_t);
+        else
+            debug(D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld) collected %ld seconds ago",
+                  rd->rrdset->rrdhost->hostname, rd->rrdset->name, rd->name, live, first_entry_t,
+                  last_entry_t, now - last_entry_t);
+        rd->state->aclk_live_status = live;
+    }
+
+    freez(claim_id);
+    return;
+}
+
 #endif //ENABLE_NEW_CLOUD_PROTOCOL
 
 // ST is read locked
diff --git a/database/sqlite/sqlite_aclk_chart.h b/database/sqlite/sqlite_aclk_chart.h
index 67d81a534e..370af4729a 100644
--- a/database/sqlite/sqlite_aclk_chart.h
+++ b/database/sqlite/sqlite_aclk_chart.h
@@ -12,8 +12,8 @@ typedef enum payload_type {
 
 extern sqlite3 *db_meta;
 
-#ifndef RRDSET_MINIMUM_LIVE_COUNT
-#define RRDSET_MINIMUM_LIVE_COUNT   3
+#ifndef RRDSET_MINIMUM_LIVE_MULTIPLIER
+#define RRDSET_MINIMUM_LIVE_MULTIPLIER   (1.5)
 #endif
 
 extern int queue_chart_to_aclk(RRDSET *st);
@@ -34,4 +34,5 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
 void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
 void aclk_process_dimension_deletion(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
 uint32_t sql_get_pending_count(struct aclk_database_worker_config *wc);
+void aclk_send_dimension_update(RRDDIM *rd);
 #endif //NETDATA_SQLITE_ACLK_CHART_H