Add support for persistent metadata (#9324)

* Implemented collector metadata logging 
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update

Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>

57 files changed, 3871 insertions, 199 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd803848ce..fe0dcb331e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES
         database/engine/pagecache.h
         database/engine/rrdenglocking.c
         database/engine/rrdenglocking.h
+        database/engine/metadata_log/metadatalog.c
+        database/engine/metadata_log/metadatalog.h
+        database/engine/metadata_log/metadatalogapi.c
+        database/engine/metadata_log/metadatalogapi.h
+        database/engine/metadata_log/logfile.h
+        database/engine/metadata_log/logfile.c
+        database/engine/metadata_log/metadatalogprotocol.h
+        database/engine/metadata_log/metalogpluginsd.c
+        database/engine/metadata_log/metalogpluginsd.h
+        database/engine/metadata_log/compaction.c
+        database/engine/metadata_log/compaction.h
+        database/engine/global_uuid_map/global_uuid_map.c
+        database/engine/global_uuid_map/global_uuid_map.h
         )
 
 set(WEB_PLUGIN_FILES
diff --git a/Makefile.am b/Makefile.am
index 889a3d8569..9f72ede368 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -375,6 +375,19 @@ if ENABLE_DBENGINE
         database/engine/pagecache.h \
         database/engine/rrdenglocking.c \
         database/engine/rrdenglocking.h \
+        database/engine/metadata_log/metadatalog.c \
+        database/engine/metadata_log/metadatalog.h \
+        database/engine/metadata_log/metadatalogapi.c \
+        database/engine/metadata_log/metadatalogapi.h \
+        database/engine/metadata_log/logfile.h \
+        database/engine/metadata_log/logfile.c \
+        database/engine/metadata_log/metadatalogprotocol.h \
+        database/engine/metadata_log/metalogpluginsd.c \
+        database/engine/metadata_log/metalogpluginsd.h \
+        database/engine/metadata_log/compaction.c \
+        database/engine/metadata_log/compaction.h \
+        database/engine/global_uuid_map/global_uuid_map.c \
+        database/engine/global_uuid_map/global_uuid_map.h \
         $(NULL)
 endif
 
diff --git a/aclk/agent_cloud_link.c b/aclk/agent_cloud_link.c
index cc366a8332..7b5aad5c4a 100644
--- a/aclk/agent_cloud_link.c
+++ b/aclk/agent_cloud_link.c
@@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi
 void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha
 void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -1752,7 +1758,7 @@ int aclk_send_info_metadata()
     debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len);
 
     buffer_sprintf(local_buffer, ", \n\t \"charts\" : ");
-    charts2json(localhost, local_buffer, 1);
+    charts2json(localhost, local_buffer, 1, 0);
     buffer_sprintf(local_buffer, "\n}\n}");
     debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len);
 
@@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd)
     UNUSED(chart_name);
     return 0;
 #else
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (!netdata_cloud_setting)
         return 0;
 
@@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
 {
     BUFFER *local_buffer = NULL;
 
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (host != localhost)
         return 0;
 
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index eab607d84f..374f7cdf1c 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                               netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_space)) {
             m->do_space = CONFIG_BOOLEAN_YES;
-            m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
+            m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
             if(unlikely(!m->st_space)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_inodes)) {
             m->do_inodes = CONFIG_BOOLEAN_YES;
-            m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
+            m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
             if(unlikely(!m->st_inodes)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
index 5cde371131..bee8395f53 100644
--- a/collectors/freebsd.plugin/plugin_freebsd.c
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(
diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c
index f253489a5b..d0b3e0fd2c 100644
--- a/collectors/macos.plugin/macos_fw.c
+++ b/collectors/macos.plugin/macos_fw.c
@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         total_disk_writes += diskstat.bytes_write;
                     }
 
-                    st = rrdset_find_bytype_localhost("disk", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
     }
 
     if (likely(do_io)) {
-        st = rrdset_find_bytype_localhost("system", "io");
+        st = rrdset_find_active_bytype_localhost("system", "io");
         if (unlikely(!st)) {
             st = rrdset_create_localhost(
                     "system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_space)) {
-                    st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_inodes)) {
-                    st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_events"
diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c
index 800b2ce56b..250186cef7 100644
--- a/collectors/macos.plugin/macos_mach_smi.c
+++ b/collectors/macos.plugin/macos_mach_smi.c
@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
                 error("DISABLED: system.cpu");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "cpu");
+                st = rrdset_find_active_bytype_localhost("system", "cpu");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"
diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c
index dddafc9f51..80b66963b5 100644
--- a/collectors/macos.plugin/macos_sysctl.c
+++ b/collectors/macos.plugin/macos_sysctl.c
@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
                 error("DISABLED: system.load");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "load");
+                st = rrdset_find_active_bytype_localhost("system", "load");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
index d8bb1b955f..fd99b35843 100644
--- a/collectors/plugins.d/plugins_d.h
+++ b/collectors/plugins.d/plugins_d.h
@@ -31,10 +31,10 @@
 #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
 #define PLUGINSD_KEYWORD_LABEL "LABEL"
 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_GUID "GUID"
-#define PLUGINSD_KEYWORD_HOST "HOST"
+#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
+#define PLUGINSD_KEYWORD_HOST "HOST"
 
 
 #define PLUGINSD_LINE_MAX 1024
diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
index 90558f7e58..ce5bf5fcf0 100644
--- a/collectors/plugins.d/pluginsd_parser.c
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION  *plugins
     return PARSER_RC_OK;
 }
 
+PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a GUID, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a GUID, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->guid_action) {
+        return plugins_action->guid_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a CONTEXT, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a CONTEXT, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->context_action) {
+        return plugins_action->context_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a TOMBSTONE, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a TOMBSTONE, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->tombstone_action) {
+        return plugins_action->tombstone_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
 
 // New plugins.d parser
 
diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h
index ea9ef40b58..ba79373cd6 100644
--- a/collectors/plugins.d/pluginsd_parser.h
+++ b/collectors/plugins.d/pluginsd_parser.h
@@ -16,19 +16,23 @@ typedef struct parser_user_object {
     struct label *new_labels;
     size_t count;
     int enabled;
+    void *private; // the user can set this for private use
 } PARSER_USER_OBJECT;
 
-PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
-PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
-PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
-           char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
-PARSER_RC pluginsd_disable_action(void *user);
-PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
-PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
-                                    RRD_ALGORITHM algorithm_type);
-PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
-PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
+extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
+extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+                                       char *title, char *units, char *plugin, char *module, int priority,
+                                       int update_every, RRDSET_TYPE