diff options
author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2020-06-12 10:35:17 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-06-12 10:35:17 +0300 |
commit | 1bd8a255441de6056a0d51a7bb787f76b590ffb6 (patch) | |
tree | 5b0ae5ca07d19e2323e202e72ca15247409b0a2d | |
parent | 45747d3e47cde80589bf702c92ff9557fe8b7f7d (diff) |
Add support for persistent metadata (#9324)
* Implemented collector metadata logging
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update
Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>
57 files changed, 3871 insertions, 199 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index dd803848ce..fe0dcb331e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES database/engine/pagecache.h database/engine/rrdenglocking.c database/engine/rrdenglocking.h + database/engine/metadata_log/metadatalog.c + database/engine/metadata_log/metadatalog.h + database/engine/metadata_log/metadatalogapi.c + database/engine/metadata_log/metadatalogapi.h + database/engine/metadata_log/logfile.h + database/engine/metadata_log/logfile.c + database/engine/metadata_log/metadatalogprotocol.h + database/engine/metadata_log/metalogpluginsd.c + database/engine/metadata_log/metalogpluginsd.h + database/engine/metadata_log/compaction.c + database/engine/metadata_log/compaction.h + database/engine/global_uuid_map/global_uuid_map.c + database/engine/global_uuid_map/global_uuid_map.h ) set(WEB_PLUGIN_FILES diff --git a/Makefile.am b/Makefile.am index 889a3d8569..9f72ede368 100644 --- a/Makefile.am +++ b/Makefile.am @@ -375,6 +375,19 @@ if ENABLE_DBENGINE database/engine/pagecache.h \ database/engine/rrdenglocking.c \ database/engine/rrdenglocking.h \ + database/engine/metadata_log/metadatalog.c \ + database/engine/metadata_log/metadatalog.h \ + database/engine/metadata_log/metadatalogapi.c \ + database/engine/metadata_log/metadatalogapi.h \ + database/engine/metadata_log/logfile.h \ + database/engine/metadata_log/logfile.c \ + database/engine/metadata_log/metadatalogprotocol.h \ + database/engine/metadata_log/metalogpluginsd.c \ + database/engine/metadata_log/metalogpluginsd.h \ + database/engine/metadata_log/compaction.c \ + database/engine/metadata_log/compaction.h \ + database/engine/global_uuid_map/global_uuid_map.c \ + database/engine/global_uuid_map/global_uuid_map.h \ $(NULL) endif diff --git a/aclk/agent_cloud_link.c b/aclk/agent_cloud_link.c index cc366a8332..7b5aad5c4a 100644 --- a/aclk/agent_cloud_link.c +++ b/aclk/agent_cloud_link.c @@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name) { struct _collector *tmp_collector; + if (unlikely(!netdata_ready)) { + return; + } COLLECTOR_LOCK; @@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name) { struct _collector *tmp_collector; + if (unlikely(!netdata_ready)) { + return; + } COLLECTOR_LOCK; @@ -1752,7 +1758,7 @@ int aclk_send_info_metadata() debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len); buffer_sprintf(local_buffer, ", \n\t \"charts\" : "); - charts2json(localhost, local_buffer, 1); + charts2json(localhost, local_buffer, 1, 0); buffer_sprintf(local_buffer, "\n}\n}"); debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len); @@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd) UNUSED(chart_name); return 0; #else + if (unlikely(!netdata_ready)) + return 0; + if (!netdata_cloud_setting) return 0; @@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae) { BUFFER *local_buffer = NULL; + if (unlikely(!netdata_ready)) + return 0; + if (host != localhost) return 0; diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index eab607d84f..374f7cdf1c 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { if(unlikely(!m->st_space)) { m->do_space = CONFIG_BOOLEAN_YES; - m->st_space = rrdset_find_bytype_localhost("disk_space", disk); + m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk); if(unlikely(!m->st_space)) { char title[4096 + 1]; snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source); @@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { if(unlikely(!m->st_inodes)) { m->do_inodes = CONFIG_BOOLEAN_YES; - m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk); + m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk); if(unlikely(!m->st_inodes)) { char title[4096 + 1]; snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source); diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c index 5cde371131..bee8395f53 100644 --- a/collectors/freebsd.plugin/plugin_freebsd.c +++ b/collectors/freebsd.plugin/plugin_freebsd.c @@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) { static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules"); + st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules"); if(!st) { st = rrdset_create_localhost( diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c index f253489a5b..d0b3e0fd2c 100644 --- a/collectors/macos.plugin/macos_fw.c +++ b/collectors/macos.plugin/macos_fw.c @@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) { total_disk_writes += diskstat.bytes_write; } - st = rrdset_find_bytype_localhost("disk", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk" @@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes); } - st = rrdset_find_bytype_localhost("disk_ops", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_ops" @@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write); } - st = rrdset_find_bytype_localhost("disk_util", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_util" @@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write); } - st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_iotime" @@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_await", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_await" @@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_avgsz" @@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_svctm" @@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) { } if (likely(do_io)) { - st = rrdset_find_bytype_localhost("system", "io"); + st = rrdset_find_active_bytype_localhost("system", "io"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" @@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------------- if (likely(do_space)) { - st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname); + st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); st = rrdset_create_localhost( @@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------------- if (likely(do_inodes)) { - st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname); + st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); st = rrdset_create_localhost( @@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net" @@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_packets" @@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_errors" @@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_drops" @@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_events" diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c index 800b2ce56b..250186cef7 100644 --- a/collectors/macos.plugin/macos_mach_smi.c +++ b/collectors/macos.plugin/macos_mach_smi.c @@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) { error("DISABLED: system.cpu"); } else { - st = rrdset_find_bytype_localhost("system", "cpu"); + st = rrdset_find_active_bytype_localhost("system", "cpu"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c index dddafc9f51..80b66963b5 100644 --- a/collectors/macos.plugin/macos_sysctl.c +++ b/collectors/macos.plugin/macos_sysctl.c @@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) { error("DISABLED: system.load"); } else { - st = rrdset_find_bytype_localhost("system", "load"); + st = rrdset_find_active_bytype_localhost("system", "load"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h index d8bb1b955f..fd99b35843 100644 --- a/collectors/plugins.d/plugins_d.h +++ b/collectors/plugins.d/plugins_d.h @@ -31,10 +31,10 @@ #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE" #define PLUGINSD_KEYWORD_LABEL "LABEL" #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE" -#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT" #define PLUGINSD_KEYWORD_GUID "GUID" -#define PLUGINSD_KEYWORD_HOST "HOST" +#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT" #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE" +#define PLUGINSD_KEYWORD_HOST "HOST" #define PLUGINSD_LINE_MAX 1024 diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 90558f7e58..ce5bf5fcf0 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins return PARSER_RC_OK; } +PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a GUID, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a GUID, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->guid_action) { + return plugins_action->guid_action(user, &uuid); + } + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a CONTEXT, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a CONTEXT, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->context_action) { + return plugins_action->context_action(user, &uuid); + } + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a TOMBSTONE, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a TOMBSTONE, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->tombstone_action) { + return plugins_action->tombstone_action(user, &uuid); + } + + return PARSER_RC_OK; +} // New plugins.d parser diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h index ea9ef40b58..ba79373cd6 100644 --- a/collectors/plugins.d/pluginsd_parser.h +++ b/collectors/plugins.d/pluginsd_parser.h @@ -16,19 +16,23 @@ typedef struct parser_user_object { struct label *new_labels; size_t count; int enabled; + void *private; // the user can set this for private use } PARSER_USER_OBJECT; -PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value); -PARSER_RC pluginsd_flush_action(void *user, RRDSET *st); -PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations); -PARSER_RC pluginsd_end_action(void *user, RRDSET *st); -PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin, - char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options); -PARSER_RC pluginsd_disable_action(void *user); -PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value); -PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options, - RRD_ALGORITHM algorithm_type); -PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source); -PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels); +extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value); +extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st); +extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations); +extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st); +extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, + char *title, char *units, char *plugin, char *module, int priority, + |