Add spinlock to protect metric release (#16989)

* Add spinlock to protect metric release
Cleanup service thread

* Adjust formatting of the shutdown message
During shutdown service thread should exit faster

4 files changed, 19 insertions, 23 deletions

diff --git a/src/daemon/main.c b/src/daemon/main.c
index 3684260710..50399db67a 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -303,7 +303,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
     do {                                                \
         usec_t now_ut = now_monotonic_usec();           \
         if(prev_msg)                                    \
-            netdata_log_info("NETDATA SHUTDOWN: in %7llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \
+            netdata_log_info("NETDATA SHUTDOWN: in %llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \
         else                                            \
             netdata_log_info("NETDATA SHUTDOWN: next: %s", msg);    \
         last_ut = now_ut;                               \
diff --git a/src/daemon/service.c b/src/daemon/service.c
index 895c858707..ff966c57d2 100644
--- a/src/daemon/service.c
+++ b/src/daemon/service.c
@@ -36,20 +36,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
 
     if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
         /* only a collector can mark a chart as obsolete, so we must remove the reference */
-
-        size_t tiers_available = 0, tiers_said_no_retention = 0;
-        for(size_t tier = 0; tier < storage_tiers ;tier++) {
-            if(rd->tiers[tier].sch) {
-                tiers_available++;
-
-                if(storage_engine_store_finalize(rd->tiers[tier].sch))
-                    tiers_said_no_retention++;
-
-                rd->tiers[tier].sch = NULL;
-            }
-        }
-
-        if (tiers_available == tiers_said_no_retention && tiers_said_no_retention) {
+        if (!rrddim_finalize_collection_and_check_retention(rd)) {
             /* This metric has no data and no references */
             metaqueue_delete_dimension_uuid(&rd->metric_uuid);
         }
@@ -204,6 +191,10 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
 
     RRDHOST *host;
     rrdhost_foreach_read(host) {
+
+        if (!service_running(SERVICE_MAINTENANCE))
+            break;
+
         if(rrdhost_receiver_replicating_charts(host) || rrdhost_sender_replicating_charts(host))
             continue;
 
@@ -321,7 +312,9 @@ void *service_main(void *ptr)
         real_step = USEC_PER_SEC;
 
         svc_rrd_cleanup_obsolete_charts_from_all_hosts();
-        svc_rrdhost_cleanup_orphan_hosts(localhost);
+
+        if (service_running(SERVICE_MAINTENANCE))
+            svc_rrdhost_cleanup_orphan_hosts(localhost);
     }
 
     netdata_thread_cleanup_pop(1);
diff --git a/src/database/rrd.h b/src/database/rrd.h
index 8ab8430ba0..dd765ad09b 100644
--- a/src/database/rrd.h
+++ b/src/database/rrd.h
@@ -260,6 +260,7 @@ typedef struct storage_collect_handle {
 struct rrddim_tier {
     STORAGE_POINT virtual_point;
     STORAGE_ENGINE_BACKEND seb;
+    SPINLOCK spinlock;
     uint32_t tier_grouping;
     time_t next_point_end_time_s;
     STORAGE_METRIC_HANDLE *smh;                     // the metric handle inside the database
diff --git a/src/database/rrddim.c b/src/database/rrddim.c
index 4f009ec053..74f48fb182 100644
--- a/src/database/rrddim.c
+++ b/src/database/rrddim.c
@@ -95,6 +95,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
             rd->tiers[tier].seb = eng->seb;
             rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
             rd->tiers[tier].smh = eng->api.metric_get_or_create(rd, host->db[tier].si);
+            rd->tiers[tier].spinlock.locked = false;
             storage_point_unset(rd->tiers[tier].virtual_point);
             initialized++;
 
@@ -169,15 +170,16 @@ bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
     size_t tiers_available = 0, tiers_said_no_retention = 0;
 
     for(size_t tier = 0; tier < storage_tiers ;tier++) {
-        if(!rd->tiers[tier].sch)
-            continue;
+        spinlock_lock(&rd->tiers[tier].spinlock);
+        if(rd->tiers[tier].sch) {
+            tiers_available++;
 
-        tiers_available++;
+            if (storage_engine_store_finalize(rd->tiers[tier].sch))
+                tiers_said_no_retention++;
 
-        if(storage_engine_store_finalize(rd->tiers[tier].sch))
-            tiers_said_no_retention++;
-
-        rd->tiers[tier].sch = NULL;
+            rd->tiers[tier].sch = NULL;
+        }
+        spinlock_unlock(&rd->tiers[tier].spinlock);
     }
 
     // return true if the dimension has retention in the db