Reset the obsolete flag on service thread (#15892)

* reset the RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS flag

* do rrdset_free even when in dbengine mode

* remove RRDSET_FLAG_ARCHIVED

* remove commented line

* use is_available_for_viewers

10 files changed, 18 insertions, 49 deletions

diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
index 9e702bc221..0a41308974 100644
--- a/collectors/plugins.d/pluginsd_parser.c
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -1604,7 +1604,7 @@ static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER
     if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_BEGIN_V2))
         return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
 
-    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE | RRDSET_FLAG_ARCHIVED)))
+    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))
         rrdset_isnot_obsolete(st);
 
     timing_step(TIMING_STEP_BEGIN2_FIND_CHART);
diff --git a/daemon/service.c b/daemon/service.c
index a25e2a26b8..f7fe86e04c 100644
--- a/daemon/service.c
+++ b/daemon/service.c
@@ -105,14 +105,11 @@ static bool svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensio
     return done_all_dimensions;
 }
 
-static void svc_rrdset_obsolete_to_archive(RRDSET *st) {
-    worker_is_busy(WORKER_JOB_ARCHIVE_CHART);
-
+static void svc_rrdset_obsolete_to_free(RRDSET *st) {
     if(!svc_rrdset_archive_obsolete_dimensions(st, true))
         return;
 
-    rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
-    rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+    worker_is_busy(WORKER_JOB_FREE_CHART);
 
     rrdcalc_unlink_all_rrdset_alerts(st);
 
@@ -130,10 +127,9 @@ static void svc_rrdset_obsolete_to_archive(RRDSET *st) {
             worker_is_busy(WORKER_JOB_SAVE_CHART);
             rrdset_save(st);
         }
-
-        worker_is_busy(WORKER_JOB_FREE_CHART);
-        rrdset_free(st);
     }
+
+    rrdset_free(st);
 }
 
 static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
@@ -150,12 +146,15 @@ static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
                      && st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now
                      && st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now
                      )) {
-            svc_rrdset_obsolete_to_archive(st);
+            svc_rrdset_obsolete_to_free(st);
         }
         else if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) {
             rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
             svc_rrdset_archive_obsolete_dimensions(st, false);
         }
+        else if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
+            rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
+        }
     }
     rrdset_foreach_done(st);
 }
diff --git a/database/contexts/instance.c b/database/contexts/instance.c
index 0c1dbf4c49..8a60ce6624 100644
--- a/database/contexts/instance.c
+++ b/database/contexts/instance.c
@@ -494,7 +494,7 @@ inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
     RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
     if(unlikely(!ri)) return;
 
-    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE)))
+    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))
         rrd_flag_set_archived(ri);
 
     rrdinstance_updated_rrdset_flags_no_action(ri, st);
diff --git a/database/rrd.h b/database/rrd.h
index baa3f4ab79..9ff838559c 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -693,9 +693,7 @@ typedef enum __attribute__ ((__packed__)) rrdset_flags {
     RRDSET_FLAG_HIDDEN                           = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting
     RRDSET_FLAG_SYNC_CLOCK                       = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
     RRDSET_FLAG_OBSOLETE_DIMENSIONS              = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions
-                                                              // No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at
-                                                              // least rrdset_free_obsolete_time seconds ago.
-    RRDSET_FLAG_ARCHIVED                         = (1 << 15),
+
     RRDSET_FLAG_METADATA_UPDATE                  = (1 << 16), // Mark that metadata needs to be stored
     RRDSET_FLAG_ANOMALY_DETECTION                = (1 << 18), // flag to identify anomaly detection charts.
     RRDSET_FLAG_INDEXED_ID                       = (1 << 19), // the rrdset is indexed by its id
@@ -1403,8 +1401,6 @@ void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
 static inline RRDSET *rrdset_find_active_localhost(const char *id)
 {
     RRDSET *st = rrdset_find_localhost(id);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1414,8 +1410,6 @@ RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
 static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
 {
     RRDSET *st = rrdset_find_bytype_localhost(type, id);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1425,8 +1419,6 @@ RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
 static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
 {
     RRDSET *st = rrdset_find_byname_localhost(name);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1442,9 +1434,8 @@ void rrdset_is_obsolete(RRDSET *st);
 void rrdset_isnot_obsolete(RRDSET *st);
 
 // checks if the RRDSET should be offered to viewers
-#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
-#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
-#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
+#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
+#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st))
 
 time_t rrddim_first_entry_s(RRDDIM *rd);
 time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier);
diff --git a/database/rrdset.c b/database/rrdset.c
index 1e00d5c8a2..285e2bfda1 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -292,11 +292,6 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
 
     ctr->react_action = RRDSET_REACT_NONE;
 
-    if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
-        rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
-        ctr->react_action |= RRDSET_REACT_CHART_ACTIVATED;
-    }
-
     if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
         ctr->react_action |= RRDSET_REACT_UPDATED;
 
@@ -657,11 +652,6 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_
 }
 
 inline void rrdset_is_obsolete(RRDSET *st) {
-    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))) {
-        netdata_log_info("Cannot obsolete already archived chart %s", rrdset_name(st));
-        return;
-    }
-
     if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
         rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
         rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
diff --git a/health/health.c b/health/health.c
index 1466453475..e04dad33b7 100644
--- a/health/health.c
+++ b/health/health.c
@@ -376,9 +376,6 @@ static void health_reload_host(RRDHOST *host) {
 
     // link the loaded alarms to their charts
     rrdset_foreach_write(st, host) {
-        if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))
-            continue;
-
         rrdcalc_link_matching_alerts_to_rrdset(st);
         rrdcalctemplate_link_matching_templates_to_rrdset(st);
     }
@@ -724,11 +721,6 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
         return 0;
     }
 
-    if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) {
-        netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rrdcalc_chart_name(rc), rrdcalc_name(rc));
-        return 0;
-    }
-
     if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
         netdata_log_debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rrdcalc_chart_name(rc), rrdcalc_name(rc));
         return 0;
@@ -854,9 +846,6 @@ static void initialize_health(RRDHOST *host)
     // link the loaded alarms to their charts
     RRDSET *st;
     rrdset_foreach_reentrant(st, host) {
-        if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))
-            continue;
-
         rrdcalc_link_matching_alerts_to_rrdset(st);
         rrdcalctemplate_link_matching_templates_to_rrdset(st);
     }
diff --git a/ml/ml.cc b/ml/ml.cc
index 867ffee7b4..9804fc343c 100644
--- a/ml/ml.cc
+++ b/ml/ml.cc
@@ -1152,7 +1152,7 @@ ml_acquired_dimension_get(char *machine_guid, STRING *chart_id, STRING *dimensio
             acq_rs = rrdset_find_and_acquire(rh, string2str(chart_id));
             if (acq_rs) {
                 RRDSET *rs = rrdset_acquired_to_rrdset(acq_rs);
-                if (rs && !rrdset_flag_check(rs, RRDSET_FLAG_ARCHIVED | RRDSET_FLAG_OBSOLETE)) {
+                if (rs && !rrdset_flag_check(rs, RRDSET_FLAG_OBSOLETE)) {
                     acq_rd = rrddim_find_and_acquire(rs, string2str(dimension_id));
                     if (acq_rd) {
                         RRDDIM *rd = rrddim_acquired_to_rrddim(acq_rd);
diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c
index 2ad068d892..61086ab635 100644
--- a/web/api/formatters/charts2json.c
+++ b/web/api/formatters/charts2json.c
@@ -36,7 +36,7 @@ const char* get_release_channel() {
     return (use_stable)?"stable":"nightly";
 }
 
-void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived) {
+void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
     static char *custom_dashboard_info_js_filename = NULL;
     size_t c, dimensions = 0, memory = 0, alarms = 0;
     RRDSET *st;
@@ -70,7 +70,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
 
     c = 0;
     rrdset_foreach_read(st, host) {
-        if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
+        if (rrdset_is_available_for_viewers(st)) {
             if(c) buffer_strcat(wb, ",");
             buffer_strcat(wb, "\n\t\t\"");
             buffer_strcat(wb, rrdset_id(st));
diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h
index 96720d4b4c..4ef6bc2c80 100644
--- a/web/api/formatters/charts2json.h
+++ b/web/api/formatters/charts2json.h
@@ -5,7 +5,7 @@
 
 #include "rrd2json.h"
 
-void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
+void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile);
 const char* get_release_channel();
 
 #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index a250c5d233..0045246f0d 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -644,7 +644,7 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
 
     buffer_flush(w->response.data);
     w->response.data->content_type = CT_APPLICATION_JSON;
-    charts2json(host, w->response.data, 0, 0);
+    charts2json(host, w->response.data, 0);
     return HTTP_RESP_OK;
 }