From 88e2c981f83d4a4cd42db6d26708bf95f54aa54f Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Sat, 16 Jul 2022 13:18:41 +0300 Subject: dont show uuids by default, added option to enable them; response is now accepting after,before to show only data for a specific timeframe; deleted items are only shown when "deleted" is requested; hub version is now shown when "queue" is requested --- database/rrdcontext.c | 123 ++++++++++++++++++++++++++++++++++--------- database/rrdcontext.h | 7 +-- web/api/netdata-swagger.json | 2 + web/api/netdata-swagger.yaml | 16 ++++++ web/api/web_api_v1.c | 16 ++++-- 5 files changed, 131 insertions(+), 33 deletions(-) diff --git a/database/rrdcontext.c b/database/rrdcontext.c index 84d35ea27a..7928099103 100644 --- a/database/rrdcontext.c +++ b/database/rrdcontext.c @@ -1860,6 +1860,8 @@ void rrdcontext_hub_stop_streaming_command(void *ptr) { struct rrdcontext_to_json { BUFFER *wb; RRDCONTEXT_TO_JSON_OPTIONS options; + time_t after; + time_t before; size_t written; time_t now; }; @@ -1869,10 +1871,18 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void * RRDMETRIC *rm = value; BUFFER *wb = t->wb; RRDCONTEXT_TO_JSON_OPTIONS options = t->options; + time_t after = t->after; + time_t before = t->before; if((rm->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)) return 0; + if(after && (!rm->last_time_t || after > rm->last_time_t)) + return 0; + + if(before && (!rm->first_time_t || before < rm->first_time_t)) + return 0; + if(t->written) buffer_strcat(wb, ",\n"); else @@ -1880,24 +1890,30 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void * buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": {", id); - char uuid[UUID_STR_LEN]; - uuid_unparse(rm->uuid, uuid); + if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) { + char uuid[UUID_STR_LEN]; + uuid_unparse(rm->uuid, uuid); + buffer_sprintf(wb, "\n\t\t\t\t\t\t\t\"uuid\":\"%s\",", uuid); + } buffer_sprintf(wb, - "\n\t\t\t\t\t\t\t\"uuid\":\"%s\"" - ",\n\t\t\t\t\t\t\t\"name\":\"%s\"" + "\n\t\t\t\t\t\t\t\"name\":\"%s\"" ",\n\t\t\t\t\t\t\t\"first_time_t\":%ld" ",\n\t\t\t\t\t\t\t\"last_time_t\":%ld" ",\n\t\t\t\t\t\t\t\"collected\":%s" - ",\n\t\t\t\t\t\t\t\"deleted\":%s" - , uuid , string2str(rm->name) , rm->first_time_t , rrd_flag_is_collected(rm) ? t->now : rm->last_time_t , rm->flags & RRD_FLAG_COLLECTED ? "true" : "false" - , rm->flags & RRD_FLAG_DELETED ? "true" : "false" ); + if(options & RRDCONTEXT_OPTION_SHOW_DELETED) { + buffer_sprintf(wb, + ",\n\t\t\t\t\t\t\t\"deleted\":%s" + , rm->flags & RRD_FLAG_DELETED ? "true" : "false" + ); + } + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) { buffer_strcat(wb, ",\n\t\t\t\t\t\t\t\"flags\":\""); rrd_flags_to_buffer(rm->flags, wb); @@ -1914,10 +1930,18 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void RRDINSTANCE *ri = value; BUFFER *wb = t->wb; RRDCONTEXT_TO_JSON_OPTIONS options = t->options; + time_t after = t->after; + time_t before = t->before; if((ri->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)) return 0; + if(after && (!ri->last_time_t || after > ri->last_time_t)) + return 0; + + if(before && (!ri->first_time_t || before < ri->first_time_t)) + return 0; + if(t->written) buffer_strcat(wb, ",\n"); else @@ -1925,12 +1949,14 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void buffer_sprintf(wb, "\t\t\t\t\"%s\": {", id); - char uuid[UUID_STR_LEN]; - uuid_unparse(ri->uuid, uuid); + if(options & RRDCONTEXT_OPTION_SHOW_UUIDS) { + char uuid[UUID_STR_LEN]; + uuid_unparse(ri->uuid, uuid); + buffer_sprintf(wb,"\n\t\t\t\t\t\"uuid\":\"%s\",", uuid); + } buffer_sprintf(wb, - "\n\t\t\t\t\t\"uuid\":\"%s\"" - ",\n\t\t\t\t\t\"name\":\"%s\"" + "\n\t\t\t\t\t\"name\":\"%s\"" ",\n\t\t\t\t\t\"context\":\"%s\"" ",\n\t\t\t\t\t\"title\":\"%s\"" ",\n\t\t\t\t\t\"units\":\"%s\"" @@ -1941,8 +1967,6 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void ",\n\t\t\t\t\t\"first_time_t\":%ld" ",\n\t\t\t\t\t\"last_time_t\":%ld" ",\n\t\t\t\t\t\"collected\":%s" - ",\n\t\t\t\t\t\"deleted\":%s" - , uuid , string2str(ri->name) , string2str(ri->rc->id) , string2str(ri->title) @@ -1954,9 +1978,15 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void , ri->first_time_t , rrd_flag_is_collected(ri) ? t->now : ri->last_time_t , ri->flags & RRD_FLAG_COLLECTED ? "true" : "false" - , ri->flags & RRD_FLAG_DELETED ? "true" : "false" ); + if(options & RRDCONTEXT_OPTION_SHOW_DELETED) { + buffer_sprintf(wb, + ",\n\t\t\t\t\t\"deleted\":%s" + , ri->flags & RRD_FLAG_DELETED ? "true" : "false" + ); + } + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) { buffer_strcat(wb, ",\n\t\t\t\t\t\"flags\":\""); rrd_flags_to_buffer(ri->flags, wb); @@ -1974,10 +2004,12 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void struct rrdcontext_to_json tt = { .wb = wb, .options = options, + .after = after, + .before = before, .written = 0, .now = t->now, }; - dictionary_sorted_walkthrough_read(ri->rrdmetrics, rrdmetric_to_json_callback, &tt); + dictionary_walkthrough_read(ri->rrdmetrics, rrdmetric_to_json_callback, &tt); buffer_strcat(wb, "\n\t\t\t\t\t}"); } @@ -1991,10 +2023,18 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void RRDCONTEXT *rc = value; BUFFER *wb = t->wb; RRDCONTEXT_TO_JSON_OPTIONS options = t->options; + time_t after = t->after; + time_t before = t->before; if((rc->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)) return 0; + if(after && (!rc->last_time_t || after > rc->last_time_t)) + return 0; + + if(before && (!rc->first_time_t || before < rc->first_time_t)) + return 0; + if(t->written) buffer_strcat(wb, ",\n"); else @@ -2008,9 +2048,7 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void rrdcontext_lock(rc); buffer_sprintf(wb, - "\n\t\t\t\"version\":%lu" - ",\n\t\t\t\"hub_version\":%lu" - ",\n\t\t\t\"title\":\"%s\"" + "\n\t\t\t\"title\":\"%s\"" ",\n\t\t\t\"units\":\"%s\"" ",\n\t\t\t\"family\":\"%s\"" ",\n\t\t\t\"chart_type\":\"%s\"" @@ -2018,9 +2056,6 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void ",\n\t\t\t\"first_time_t\":%ld" ",\n\t\t\t\"last_time_t\":%ld" ",\n\t\t\t\"collected\":%s" - ",\n\t\t\t\"deleted\":%s" - , rc->version - , rc->hub.version , string2str(rc->title) , string2str(rc->units) , string2str(rc->family) @@ -2029,9 +2064,15 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void , rc->first_time_t , rrd_flag_is_collected(rc) ? t->now : rc->last_time_t , rc->flags & RRD_FLAG_COLLECTED ? "true" : "false" - , rc->flags & RRD_FLAG_DELETED ? "true" : "false" ); + if(options & RRDCONTEXT_OPTION_SHOW_DELETED) { + buffer_sprintf(wb, + ",\n\t\t\t\"deleted\":%s" + , rc->flags & RRD_FLAG_DELETED ? "true" : "false" + ); + } + if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) { buffer_strcat(wb, ",\n\t\t\t\"flags\":\""); rrd_flags_to_buffer(rc->flags, wb); @@ -2047,9 +2088,13 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void ",\n\t\t\t\"last_queued\":%llu" ",\n\t\t\t\"scheduled_dispatch\":%llu" ",\n\t\t\t\"last_dequeued\":%llu" + ",\n\t\t\t\"hub_version\":%lu" + ",\n\t\t\t\"version\":%lu" , rc->queue.queued_ut / USEC_PER_SEC , rc->queue.scheduled_dispatch_ut / USEC_PER_SEC , rc->queue.dequeued_ut / USEC_PER_SEC + , rc->hub.version + , rc->version ); } @@ -2060,10 +2105,12 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void struct rrdcontext_to_json tt = { .wb = wb, .options = options, + .after = after, + .before = before, .written = 0, .now = t->now, }; - dictionary_sorted_walkthrough_read(rc->rrdinstances, rrdinstance_to_json_callback, &tt); + dictionary_walkthrough_read(rc->rrdinstances, rrdinstance_to_json_callback, &tt); buffer_strcat(wb, "\n\t\t\t}"); } @@ -2072,7 +2119,7 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void return 1; } -int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context) { +int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context) { RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, context); if(!rca) return HTTP_RESP_NOT_FOUND; @@ -2081,19 +2128,33 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS opt if(options & RRDCONTEXT_OPTION_DEEPSCAN) rrdcontext_recalculate_context_retention(rc, RRD_FLAG_NONE, -1); + if(after != 0 && before != 0) { + long long after_wanted = after; + long long before_wanted = before; + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted); + after = after_wanted; + before = before_wanted; + } + struct rrdcontext_to_json t = { .wb = wb, .options = options|RRDCONTEXT_OPTION_SKIP_ID, + .after = after, + .before = before, .written = 0, .now = now_realtime_sec(), }; rrdcontext_to_json_callback(context, rc, &t); rrdcontext_release(rca); + + if(!t.written) + return HTTP_RESP_NOT_FOUND; + return HTTP_RESP_OK; } -int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS options) { +int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options) { char node_uuid[UUID_STR_LEN] = ""; if(host->node_id) @@ -2102,6 +2163,14 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS op if(options & RRDCONTEXT_OPTION_DEEPSCAN) rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, -1); + if(after != 0 && before != 0) { + long long after_wanted = after; + long long before_wanted = before; + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted); + after = after_wanted; + before = before_wanted; + } + buffer_sprintf(wb, "{\n" "\t\"hostname\": \"%s\"" ",\n\t\"machine_guid\": \"%s\"" @@ -2123,10 +2192,12 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS op struct rrdcontext_to_json t = { .wb = wb, .options = options, + .after = after, + .before = before, .written = 0, .now = now_realtime_sec(), }; - dictionary_sorted_walkthrough_read((DICTIONARY *)host->rrdctx, rrdcontext_to_json_callback, &t); + dictionary_walkthrough_read((DICTIONARY *)host->rrdctx, rrdcontext_to_json_callback, &t); // close contexts, close main buffer_strcat(wb, "\n\t}\n}"); diff --git a/database/rrdcontext.h b/database/rrdcontext.h index 7e56159523..d26ab8c94b 100644 --- a/database/rrdcontext.h +++ b/database/rrdcontext.h @@ -43,13 +43,14 @@ typedef enum { RRDCONTEXT_OPTION_SHOW_FLAGS = (1 << 4), RRDCONTEXT_OPTION_SHOW_DELETED = (1 << 5), RRDCONTEXT_OPTION_DEEPSCAN = (1 << 6), + RRDCONTEXT_OPTION_SHOW_UUIDS = (1 << 7), RRDCONTEXT_OPTION_SKIP_ID = (1 << 31), // internal use } RRDCONTEXT_TO_JSON_OPTIONS; -#define RRDCONTEXT_OPTIONS_ALL (RRDCONTEXT_OPTION_SHOW_METRICS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_QUEUED|RRDCONTEXT_OPTION_SHOW_FLAGS|RRDCONTEXT_OPTION_SHOW_DELETED) +#define RRDCONTEXT_OPTIONS_ALL (RRDCONTEXT_OPTION_SHOW_METRICS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_QUEUED|RRDCONTEXT_OPTION_SHOW_FLAGS|RRDCONTEXT_OPTION_SHOW_DELETED|RRDCONTEXT_OPTION_SHOW_UUIDS) -extern int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context); -extern int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, RRDCONTEXT_TO_JSON_OPTIONS options); +extern int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context); +extern int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options); // ---------------------------------------------------------------------------- // public API for rrddims diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json index fecb17473f..37c754024a 100644 --- a/web/api/netdata-swagger.json +++ b/web/api/netdata-swagger.json @@ -103,6 +103,7 @@ "charts", "dimensions", "labels", + "uuids", "queue", "flags", "deleted", @@ -161,6 +162,7 @@ "charts", "dimensions", "labels", + "uuids", "queue", "flags", "deleted", diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml index 771fc93aec..e8ee69207c 100644 --- a/web/api/netdata-swagger.yaml +++ b/web/api/netdata-swagger.yaml @@ -85,6 +85,7 @@ paths: - charts - dimensions - labels + - uuids - queue - flags - deleted @@ -126,12 +127,27 @@ paths: - charts - dimensions - labels + - uuids - queue - flags - deleted - deepscan default: - full + - name: after + in: query + description: limit the results on context having data after this timestamp. + required: false + schema: + type: number + format: integer + - name: before + in: query + description: limit the results on context having data before this timestamp. + required: false + schema: + type: number + format: integer responses: "200": description: A javascript object with detailed information about the context. diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index bc851751f1..d9205f75a3 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -391,6 +391,8 @@ static RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) { options |= RRDCONTEXT_OPTION_SHOW_QUEUED; else if(!strcmp(tok, "flags")) options |= RRDCONTEXT_OPTION_SHOW_FLAGS; + else if(!strcmp(tok, "uuids")) + options |= RRDCONTEXT_OPTION_SHOW_UUIDS; else if(!strcmp(tok, "deleted")) options |= RRDCONTEXT_OPTION_SHOW_DELETED; else if(!strcmp(tok, "labels")) @@ -405,6 +407,7 @@ static RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) { static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w, char *url) { char *context = NULL; RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE; + time_t after = 0, before = 0; buffer_flush(w->response.data); @@ -420,7 +423,9 @@ static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w // they are not null and not empty if(!strcmp(name, "context") || !strcmp(name, "ctx")) context = value; - if(!strcmp(name, "options")) options = rrdcontext_to_json_parse_options(value); + else if(!strcmp(name, "after")) after = str2l(value); + else if(!strcmp(name, "before")) before = str2l(value); + else if(!strcmp(name, "options")) options = rrdcontext_to_json_parse_options(value); } if(!context || !*context) { @@ -430,11 +435,12 @@ static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w buffer_flush(w->response.data); w->response.data->contenttype = CT_APPLICATION_JSON; - return rrdcontext_to_json(host, w->response.data, options, context); + return rrdcontext_to_json(host, w->response.data, after, before, options, context); } static int web_client_api_request_v1_contexts(RRDHOST *host, struct web_client *w, char *url) { RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE; + time_t after = 0, before = 0; buffer_flush(w->response.data); @@ -449,11 +455,13 @@ static int web_client_api_request_v1_contexts(RRDHOST *host, struct web_client * // name and value are now the parameters // they are not null and not empty - if(!strcmp(name, "options")) options = rrdcontext_to_json_parse_options(value); + if(!strcmp(name, "after")) after = str2l(value); + else if(!strcmp(name, "before")) before = str2l(value); + else if(!strcmp(name, "options")) options = rrdcontext_to_json_parse_options(value); } w->response.data->contenttype = CT_APPLICATION_JSON; - return rrdcontexts_to_json(host, w->response.data, options); + return rrdcontexts_to_json(host, w->response.data, after, before, options); } inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url) { -- cgit v1.2.3