Add support for persistent metadata (#9324)

* Implemented collector metadata logging 
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update

Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>

11 files changed, 110 insertions, 37 deletions

diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index eab607d84f..374f7cdf1c 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                               netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_space)) {
             m->do_space = CONFIG_BOOLEAN_YES;
-            m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
+            m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
             if(unlikely(!m->st_space)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_inodes)) {
             m->do_inodes = CONFIG_BOOLEAN_YES;
-            m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
+            m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
             if(unlikely(!m->st_inodes)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
index 5cde371131..bee8395f53 100644
--- a/collectors/freebsd.plugin/plugin_freebsd.c
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(
diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c
index f253489a5b..d0b3e0fd2c 100644
--- a/collectors/macos.plugin/macos_fw.c
+++ b/collectors/macos.plugin/macos_fw.c
@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         total_disk_writes += diskstat.bytes_write;
                     }
 
-                    st = rrdset_find_bytype_localhost("disk", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
     }
 
     if (likely(do_io)) {
-        st = rrdset_find_bytype_localhost("system", "io");
+        st = rrdset_find_active_bytype_localhost("system", "io");
         if (unlikely(!st)) {
             st = rrdset_create_localhost(
                     "system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_space)) {
-                    st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_inodes)) {
-                    st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_events"
diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c
index 800b2ce56b..250186cef7 100644
--- a/collectors/macos.plugin/macos_mach_smi.c
+++ b/collectors/macos.plugin/macos_mach_smi.c
@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
                 error("DISABLED: system.cpu");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "cpu");
+                st = rrdset_find_active_bytype_localhost("system", "cpu");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"
diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c
index dddafc9f51..80b66963b5 100644
--- a/collectors/macos.plugin/macos_sysctl.c
+++ b/collectors/macos.plugin/macos_sysctl.c
@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
                 error("DISABLED: system.load");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "load");
+                st = rrdset_find_active_bytype_localhost("system", "load");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
index d8bb1b955f..fd99b35843 100644
--- a/collectors/plugins.d/plugins_d.h
+++ b/collectors/plugins.d/plugins_d.h
@@ -31,10 +31,10 @@
 #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
 #define PLUGINSD_KEYWORD_LABEL "LABEL"
 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_GUID "GUID"
-#define PLUGINSD_KEYWORD_HOST "HOST"
+#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
+#define PLUGINSD_KEYWORD_HOST "HOST"
 
 
 #define PLUGINSD_LINE_MAX 1024
diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
index 90558f7e58..ce5bf5fcf0 100644
--- a/collectors/plugins.d/pluginsd_parser.c
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION  *plugins
     return PARSER_RC_OK;
 }
 
+PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a GUID, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a GUID, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->guid_action) {
+        return plugins_action->guid_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a CONTEXT, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a CONTEXT, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->context_action) {
+        return plugins_action->context_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a TOMBSTONE, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a TOMBSTONE, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->tombstone_action) {
+        return plugins_action->tombstone_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
 
 // New plugins.d parser
 
diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h
index ea9ef40b58..ba79373cd6 100644
--- a/collectors/plugins.d/pluginsd_parser.h
+++ b/collectors/plugins.d/pluginsd_parser.h
@@ -16,19 +16,23 @@ typedef struct parser_user_object {
     struct label *new_labels;
     size_t count;
     int enabled;
+    void *private; // the user can set this for private use
 } PARSER_USER_OBJECT;
 
-PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
-PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
-PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
-           char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
-PARSER_RC pluginsd_disable_action(void *user);
-PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
-PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
-                                    RRD_ALGORITHM algorithm_type);
-PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
-PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
+extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
+extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+                                       char *title, char *units, char *plugin, char *module, int priority,
+                                       int update_every, RRDSET_TYPE chart_type, char *options);
+extern PARSER_RC pluginsd_disable_action(void *user);
+extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global,
+                                          calculated_number value);
+extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+                                           long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
+extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
+extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+
 
 #endif //NETDATA_PLUGINSD_PARSER_H
diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c
index c9eef2c257..1334f5d210 100644
--- a/collectors/proc.plugin/plugin_proc.c
+++ b/collectors/proc.plugin/plugin_proc.c
@@ -148,7 +148,7 @@ void *proc_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(
diff --git a/collectors/proc.plugin/proc_net_softnet_stat.c b/collectors/proc.plugin/proc_net_softnet_stat.c
index 7ec783e77d..a29ccccd1a 100644
--- a/collectors/proc.plugin/proc_net_softnet_stat.c
+++ b/collectors/proc.plugin/proc_net_softnet_stat.c
@@ -81,7 +81,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
 
     // --------------------------------------------------------------------
 
-    st = rrdset_find_bytype_localhost("system", "softnet_stat");
+    st = rrdset_find_active_bytype_localhost("system", "softnet_stat");
     if(unlikely(!st)) {
         st = rrdset_create_localhost(
                 "system"
@@ -114,7 +114,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
             char id[50+1];
             snprintfz(id, 50, "cpu%zu_softnet_stat", l);
 
-            st = rrdset_find_bytype_localhost("cpu", id);
+            st = rrdset_find_active_bytype_localhost("cpu", id);
             if(unlikely(!st)) {
                 char title[100+1];
                 snprintfz(title, 100, "CPU%zu softnet_stat", l);
diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c
index df69031dea..1e69d53460 100644
--- a/collectors/statsd.plugin/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -1461,6 +1461,8 @@ static inline RRDSET *statsd_private_rrdset_create(
             , chart_type      // chart type
             , memory_mode     // memory mode
             , history         // history
+            , 0               // not archived
+            , NULL            // no known UUID
     );
     rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
 
@@ -1999,6 +2001,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
                 , chart->chart_type         // chart type
                 , app->rrd_memory_mode      // memory mode
                 , app->rrd_history_entries  // history
+                , 0                         // not archived
+                , NULL                      // no known UUID
         );
 
         rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);