diff options
author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2020-06-12 10:35:17 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-06-12 10:35:17 +0300 |
commit | 1bd8a255441de6056a0d51a7bb787f76b590ffb6 (patch) | |
tree | 5b0ae5ca07d19e2323e202e72ca15247409b0a2d /collectors | |
parent | 45747d3e47cde80589bf702c92ff9557fe8b7f7d (diff) |
Add support for persistent metadata (#9324)
* Implemented collector metadata logging
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update
Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>
Diffstat (limited to 'collectors')
-rw-r--r-- | collectors/diskspace.plugin/plugin_diskspace.c | 4 | ||||
-rw-r--r-- | collectors/freebsd.plugin/plugin_freebsd.c | 2 | ||||
-rw-r--r-- | collectors/macos.plugin/macos_fw.c | 30 | ||||
-rw-r--r-- | collectors/macos.plugin/macos_mach_smi.c | 2 | ||||
-rw-r--r-- | collectors/macos.plugin/macos_sysctl.c | 2 | ||||
-rw-r--r-- | collectors/plugins.d/plugins_d.h | 4 | ||||
-rw-r--r-- | collectors/plugins.d/pluginsd_parser.c | 65 | ||||
-rw-r--r-- | collectors/plugins.d/pluginsd_parser.h | 28 | ||||
-rw-r--r-- | collectors/proc.plugin/plugin_proc.c | 2 | ||||
-rw-r--r-- | collectors/proc.plugin/proc_net_softnet_stat.c | 4 | ||||
-rw-r--r-- | collectors/statsd.plugin/statsd.c | 4 |
11 files changed, 110 insertions, 37 deletions
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index eab607d84f..374f7cdf1c 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { if(unlikely(!m->st_space)) { m->do_space = CONFIG_BOOLEAN_YES; - m->st_space = rrdset_find_bytype_localhost("disk_space", disk); + m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk); if(unlikely(!m->st_space)) { char title[4096 + 1]; snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source); @@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) { if(unlikely(!m->st_inodes)) { m->do_inodes = CONFIG_BOOLEAN_YES; - m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk); + m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk); if(unlikely(!m->st_inodes)) { char title[4096 + 1]; snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source); diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c index 5cde371131..bee8395f53 100644 --- a/collectors/freebsd.plugin/plugin_freebsd.c +++ b/collectors/freebsd.plugin/plugin_freebsd.c @@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) { static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules"); + st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules"); if(!st) { st = rrdset_create_localhost( diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c index f253489a5b..d0b3e0fd2c 100644 --- a/collectors/macos.plugin/macos_fw.c +++ b/collectors/macos.plugin/macos_fw.c @@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) { total_disk_writes += diskstat.bytes_write; } - st = rrdset_find_bytype_localhost("disk", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk" @@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes); } - st = rrdset_find_bytype_localhost("disk_ops", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_ops" @@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write); } - st = rrdset_find_bytype_localhost("disk_util", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_util" @@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) { CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write); } - st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_iotime" @@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_await", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_await" @@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_avgsz" @@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name); + st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name); if (unlikely(!st)) { st = rrdset_create_localhost( "disk_svctm" @@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) { } if (likely(do_io)) { - st = rrdset_find_bytype_localhost("system", "io"); + st = rrdset_find_active_bytype_localhost("system", "io"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" @@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------------- if (likely(do_space)) { - st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname); + st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); st = rrdset_create_localhost( @@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------------- if (likely(do_inodes)) { - st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname); + st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); st = rrdset_create_localhost( @@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net" @@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_packets" @@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_errors" @@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_drops" @@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name); + st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name); if (unlikely(!st)) { st = rrdset_create_localhost( "net_events" diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c index 800b2ce56b..250186cef7 100644 --- a/collectors/macos.plugin/macos_mach_smi.c +++ b/collectors/macos.plugin/macos_mach_smi.c @@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) { error("DISABLED: system.cpu"); } else { - st = rrdset_find_bytype_localhost("system", "cpu"); + st = rrdset_find_active_bytype_localhost("system", "cpu"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c index dddafc9f51..80b66963b5 100644 --- a/collectors/macos.plugin/macos_sysctl.c +++ b/collectors/macos.plugin/macos_sysctl.c @@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) { error("DISABLED: system.load"); } else { - st = rrdset_find_bytype_localhost("system", "load"); + st = rrdset_find_active_bytype_localhost("system", "load"); if (unlikely(!st)) { st = rrdset_create_localhost( "system" diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h index d8bb1b955f..fd99b35843 100644 --- a/collectors/plugins.d/plugins_d.h +++ b/collectors/plugins.d/plugins_d.h @@ -31,10 +31,10 @@ #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE" #define PLUGINSD_KEYWORD_LABEL "LABEL" #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE" -#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT" #define PLUGINSD_KEYWORD_GUID "GUID" -#define PLUGINSD_KEYWORD_HOST "HOST" +#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT" #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE" +#define PLUGINSD_KEYWORD_HOST "HOST" #define PLUGINSD_LINE_MAX 1024 diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 90558f7e58..ce5bf5fcf0 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins return PARSER_RC_OK; } +PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a GUID, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a GUID, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->guid_action) { + return plugins_action->guid_action(user, &uuid); + } + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a CONTEXT, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a CONTEXT, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->context_action) { + return plugins_action->context_action(user, &uuid); + } + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action) +{ + char *uuid_str = words[1]; + uuid_t uuid; + + if (unlikely(!uuid_str)) { + error("requested a TOMBSTONE, without a uuid."); + return PARSER_RC_ERROR; + } + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) { + error("requested a TOMBSTONE, without a valid uuid string."); + return PARSER_RC_ERROR; + } + + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str); + if (plugins_action->tombstone_action) { + return plugins_action->tombstone_action(user, &uuid); + } + + return PARSER_RC_OK; +} // New plugins.d parser diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h index ea9ef40b58..ba79373cd6 100644 --- a/collectors/plugins.d/pluginsd_parser.h +++ b/collectors/plugins.d/pluginsd_parser.h @@ -16,19 +16,23 @@ typedef struct parser_user_object { struct label *new_labels; size_t count; int enabled; + void *private; // the user can set this for private use } PARSER_USER_OBJECT; -PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value); -PARSER_RC pluginsd_flush_action(void *user, RRDSET *st); -PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations); -PARSER_RC pluginsd_end_action(void *user, RRDSET *st); -PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin, - char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options); -PARSER_RC pluginsd_disable_action(void *user); -PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value); -PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options, - RRD_ALGORITHM algorithm_type); -PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source); -PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels); +extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value); +extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st); +extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations); +extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st); +extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, + char *title, char *units, char *plugin, char *module, int priority, + int update_every, RRDSET_TYPE chart_type, char *options); +extern PARSER_RC pluginsd_disable_action(void *user); +extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, + calculated_number value); +extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, + long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type); +extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source); +extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels); + #endif //NETDATA_PLUGINSD_PARSER_H diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c index c9eef2c257..1334f5d210 100644 --- a/collectors/proc.plugin/plugin_proc.c +++ b/collectors/proc.plugin/plugin_proc.c @@ -148,7 +148,7 @@ void *proc_main(void *ptr) { static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules"); + st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules"); if(!st) { st = rrdset_create_localhost( diff --git a/collectors/proc.plugin/proc_net_softnet_stat.c b/collectors/proc.plugin/proc_net_softnet_stat.c index 7ec783e77d..a29ccccd1a 100644 --- a/collectors/proc.plugin/proc_net_softnet_stat.c +++ b/collectors/proc.plugin/proc_net_softnet_stat.c @@ -81,7 +81,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { // -------------------------------------------------------------------- - st = rrdset_find_bytype_localhost("system", "softnet_stat"); + st = rrdset_find_active_bytype_localhost("system", "softnet_stat"); if(unlikely(!st)) { st = rrdset_create_localhost( "system" @@ -114,7 +114,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { char id[50+1]; snprintfz(id, 50, "cpu%zu_softnet_stat", l); - st = rrdset_find_bytype_localhost("cpu", id); + st = rrdset_find_active_bytype_localhost("cpu", id); if(unlikely(!st)) { char title[100+1]; snprintfz(title, 100, "CPU%zu softnet_stat", l); diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c index df69031dea..1e69d53460 100644 --- a/collectors/statsd.plugin/statsd.c +++ b/collectors/statsd.plugin/statsd.c @@ -1461,6 +1461,8 @@ static inline RRDSET *statsd_private_rrdset_create( , chart_type // chart type , memory_mode // memory mode , history // history + , 0 // not archived + , NULL // no known UUID ); rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST); @@ -1999,6 +2001,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch , chart->chart_type // chart type , app->rrd_memory_mode // memory mode , app->rrd_history_entries // history + , 0 // not archived + , NULL // no known UUID ); rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST); |