diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-02-15 21:16:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-15 21:16:29 +0200 |
commit | d2daa19bf53c9a8cb781c8e50a86b9961b0503a9 (patch) | |
tree | 8d8b744138c28e010a24456aee55447d31a719bd /web/api/formatters/json_wrapper.c | |
parent | 37a918ae2bc996fc881ab60042ae5a8f434f4c52 (diff) |
JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)
* first work on standardizing json formatting
* renamed old grouping to time_grouping and added group_by
* add dummy functions to enable compilation
* buffer json api work
* jsonwrap opening with buffer_json_X() functions
* cleanup
* storage for quotes
* optimize buffer printing for both numbers and strings
* removed ; from define
* contexts json generation using the new json functions
* fix buffer overflow at unit test
* weights endpoint using new json api
* fixes to weights endpoint
* check buffer overflow on all buffer functions
* do synchronous queries for weights
* buffer_flush() now resets json state too
* content type typedef
* print double values that are above the max 64-bit value
* str2ndd() can now parse values above UINT64_MAX
* faster number parsing by avoiding double calculations as much as possible
* faster number parsing
* faster hex parsing
* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers
* full printing and parsing without using library functions - and related unit tests
* added IEEE754 streaming capability to enable streaming of double values in hex
* streaming and replication to transfer all values in hex
* use our own str2ndd for set2
* remove subnormal check from ieee
* base64 encoding for numbers, instead of hex
* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision
* str2ndd_encoded() parses all encoding formats, including integers
* prevent uninitialized use
* /api/v1/info using the new json API
* Fix error when compiling with --disable-ml
* Remove redundant 'buffer_unittest' declaration
* Fix formatting
* Fix formatting
* Fix formatting
* fix buffer unit test
* apps.plugin using the new JSON API
* make sure the metrics registry does not accept negative timestamps
* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache
* Fix more formatting
---------
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Diffstat (limited to 'web/api/formatters/json_wrapper.c')
-rw-r--r-- | web/api/formatters/json_wrapper.c | 668 |
1 files changed, 309 insertions, 359 deletions
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index aa663495a7..ba71ccc86d 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -2,433 +2,388 @@ #include "json_wrapper.h" -struct value_output { - int c; - BUFFER *wb; -}; - -static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { - struct value_output *ap = (struct value_output *)data; - BUFFER *wb = ap->wb; - char *output = (char *) entry; - if(ap->c) buffer_strcat(wb, ","); - buffer_strcat(wb, output); - (ap->c)++; - return 0; -} - -static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { - (void)ls; - DICTIONARY *dict = (DICTIONARY *)data; - char n[RRD_ID_LENGTH_MAX * 2 + 2]; - char output[RRD_ID_LENGTH_MAX * 2 + 8]; - char v[RRD_ID_LENGTH_MAX * 2 + 1]; - - sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v); - snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v); - dictionary_set(dict, n, output, len + 1); - - return 1; -} - -void rrdr_show_plan(RRDR *r, BUFFER *wb, const char *kq, const char *sq __maybe_unused) { +void jsonwrap_query_plan(RRDR *r, BUFFER *wb) { QUERY_TARGET *qt = r->internal.qt; - buffer_sprintf(wb, "\n\t%squery_plan%s: {", kq, kq); - + buffer_json_member_add_object(wb, "query_plan"); for(size_t m = 0; m < qt->query.used; m++) { QUERY_METRIC *qm = &qt->query.array[m]; - if(m) - buffer_strcat(wb, ","); - - buffer_sprintf(wb, "\n\t\t%s%s%s: {", kq, string2str(qm->dimension.id), kq); - - buffer_sprintf(wb, "\n\t\t\t%splans%s: [", kq, kq); - for(size_t p = 0; p < qm->plan.used ;p++) { - QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; - if(p) - buffer_strcat(wb, ","); - - buffer_strcat(wb, "\n\t\t\t\t{"); - buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, qp->tier); - buffer_sprintf(wb, "\n\t\t\t\t\t%safter%s: %ld,", kq, kq, qp->after); - buffer_sprintf(wb, "\n\t\t\t\t\t%sbefore%s: %ld", kq, kq, qp->before); - buffer_strcat(wb, "\n\t\t\t\t}"); - } - buffer_strcat(wb, "\n\t\t\t],"); - - buffer_sprintf(wb, "\n\t\t\t%stiers%s: [", kq, kq); - for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(tier) - buffer_strcat(wb, ","); - - buffer_strcat(wb, "\n\t\t\t\t{"); - buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, tier); - buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_first_time%s: %ld,", kq, kq, qm->tiers[tier].db_first_time_s); - buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_last_time%s: %ld,", kq, kq, qm->tiers[tier].db_last_time_s); - buffer_sprintf(wb, "\n\t\t\t\t\t%sweight%s: %ld", kq, kq, qm->tiers[tier].weight); - buffer_strcat(wb, "\n\t\t\t\t}"); + buffer_json_member_add_object(wb, string2str(qm->dimension.id)); + { + buffer_json_member_add_array(wb, "plans"); + for (size_t p = 0; p < qm->plan.used; p++) { + QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tier", qp->tier); + buffer_json_member_add_time_t(wb, "after", qp->after); + buffer_json_member_add_time_t(wb, "before", qp->before); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "tiers"); + for (size_t tier = 0; tier < storage_tiers; tier++) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tier", tier); + buffer_json_member_add_time_t(wb, "db_first_time", qm->tiers[tier].db_first_time_s); + buffer_json_member_add_time_t(wb, "db_last_time", qm->tiers[tier].db_last_time_s); + buffer_json_member_add_int64(wb, "weight", qm->tiers[tier].weight); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); } - buffer_strcat(wb, "\n\t\t\t]"); - - buffer_strcat(wb, "\n\t\t}"); + buffer_json_object_close(wb); } - - buffer_strcat(wb, "\n\t},"); + buffer_json_object_close(wb); } -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - RRDR_GROUPING group_method) -{ +static inline long jsonwrap_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { QUERY_TARGET *qt = r->internal.qt; - - long rows = rrdr_rows(r); - long c, i; const long query_used = qt->query.used; + long c, i; - //info("JSONWRAPPER(): %s: BEGIN", r->st->id); - char kq[2] = "", // key quote - sq[2] = ""; // string quote - - if( options & RRDR_OPTION_GOOGLE_JSON ) { - kq[0] = '\0'; - sq[0] = '\''; - } - else { - kq[0] = '"'; - sq[0] = '"'; - } - - buffer_sprintf(wb, "{\n" - " %sapi%s: 1,\n" - " %sid%s: %s%s%s,\n" - " %sname%s: %s%s%s,\n" - " %sview_update_every%s: %lld,\n" - " %supdate_every%s: %lld,\n" - " %sfirst_entry%s: %lld,\n" - " %slast_entry%s: %lld,\n" - " %sbefore%s: %lld,\n" - " %safter%s: %lld,\n" - " %sgroup%s: %s%s%s,\n" - " %soptions%s: %s" - , kq, kq - , kq, kq, sq, qt->id, sq - , kq, kq, sq, qt->id, sq - , kq, kq, (long long)r->update_every - , kq, kq, (long long)qt->db.minimum_latest_update_every_s - , kq, kq, (long long)qt->db.first_time_s - , kq, kq, (long long)qt->db.last_time_s - , kq, kq, (long long)r->before - , kq, kq, (long long)r->after - , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq - , kq, kq, sq); - - web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options); - - buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); - + buffer_json_member_add_array(wb, key); for(c = 0, i = 0; c < query_used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - - if(i) buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); - buffer_strcat(wb, sq); + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.name)); i++; } - if(!i) { -#ifdef NETDATA_INTERNAL_CHECKS - error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options); -#endif - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); - } + buffer_json_array_close(wb); - buffer_sprintf(wb, "],\n" - " %sdimension_ids%s: [" - , kq, kq); + return i; +} + +static inline long jsonwrap_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; + buffer_json_member_add_array(wb, key); for(c = 0, i = 0; c < query_used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - - if(i) buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.id)); - buffer_strcat(wb, sq); + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(qt->query.array[c].dimension.id)); i++; } - if(!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); - } - buffer_strcat(wb, "],\n"); + buffer_json_array_close(wb); - if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) { - buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq); + return i; +} - char name[RRD_ID_LENGTH_MAX * 2 + 2]; - char output[RRD_ID_LENGTH_MAX * 2 + 8]; +static inline long jsonwrap_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; - struct value_output co = {.c = 0, .wb = wb}; + buffer_json_member_add_array(wb, key); + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->metrics.used ;c++) { - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdmetric_acquired_id(qt->metrics.array[c]), - rrdmetric_acquired_name(qt->metrics.array[c])); + QUERY_METRIC *qm = &qt->query.array[c]; + buffer_json_add_array_item_string(wb, string2str(qm->chart.id)); + i++; + } + buffer_json_array_close(wb); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", - rrdmetric_acquired_id(qt->metrics.array[c]), - rrdmetric_acquired_name(qt->metrics.array[c])); + return i; +} - dictionary_set(dict, name, output, len + 1); - } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); +struct rrdlabels_formatting_v2 { + BUFFER *wb; + DICTIONARY *dict; +}; - co.c = 0; - buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->instances.used ; c++) { - RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; +static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { + struct rrdlabels_formatting_v2 *t = data; + BUFFER *wb = t->wb; + DICTIONARY *dict = t->dict; - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdinstance_acquired_id(ria), - rrdinstance_acquired_name(ria)); + char n[RRD_ID_LENGTH_MAX * 2 + 2]; + snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value); + + bool existing = 0; + bool *set = dictionary_set(dict, n, &existing, sizeof(bool)); + if(!*set) { + *set = true; + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, name); + buffer_json_add_array_item_string(wb, value); + buffer_json_array_close(wb); + } + + return 1; +} - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", - rrdinstance_acquired_id(ria), - rrdinstance_acquired_name(ria)); +static inline void jsonwrap_full_dimension_list(BUFFER *wb, RRDR *r) { + QUERY_TARGET *qt = r->internal.qt; - dictionary_set(dict, name, output, len + 1); + char name[RRD_ID_LENGTH_MAX * 2 + 2]; + + buffer_json_member_add_array(wb, "full_dimension_list"); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long)qt->metrics.used ;c++) { + RRDMETRIC_ACQUIRED *rma = qt->metrics.array[c]; + + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdmetric_acquired_id(rma), + rrdmetric_acquired_name(rma)); + + bool existing = 0; + bool *set = dictionary_set(dict, name, &existing, sizeof(bool)); + if(!*set) { + *set = true; + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, rrdmetric_acquired_id(rma)); + buffer_json_add_array_item_string(wb, rrdmetric_acquired_name(rma)); + buffer_json_array_close(wb); } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); - - co.c = 0; - buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->instances.used ; c++) { - RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; - rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict); + } + dictionary_destroy(dict); + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "full_chart_list"); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long)qt->instances.used ; c++) { + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; + + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdinstance_acquired_id(ria), + rrdinstance_acquired_name(ria)); + + bool existing = 0; + bool *set = dictionary_set(dict, name, &existing, sizeof(bool)); + if(!*set) { + *set = true; + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(ria)); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(ria)); + buffer_json_array_close(wb); } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); - buffer_strcat(wb, "],\n"); } + dictionary_destroy(dict); + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "full_chart_labels"); + struct rrdlabels_formatting_v2 t = { + .wb = wb, + .dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE), + }; + for (long c = 0; c < (long)qt->instances.used ; c++) { + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t); + } + dictionary_destroy(t.dict); + buffer_json_array_close(wb); +} - // functions - { - DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - RRDINSTANCE_ACQUIRED *ria = NULL; - for (c = 0; c < query_used ; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; - if(qm->link.ria == ria) - continue; +static inline void jsonwrap_functions(BUFFER *wb, const char *key, RRDR *r) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; - ria = qm->link.ria; - chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs); - } + DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + RRDINSTANCE_ACQUIRED *ria = NULL; + for (long c = 0; c < query_used ; c++) { + QUERY_METRIC *qm = &qt->query.array[c]; + if(qm->link.ria == ria) + continue; - buffer_sprintf(wb, " %sfunctions%s: [", kq, kq); - void *t; (void)t; - dfe_start_read(funcs, t) { - const char *comma = ""; - if(t_dfe.counter) comma = ", "; - buffer_sprintf(wb, "%s%s%s%s", comma, sq, t_dfe.name, sq); - } - dfe_done(t); - dictionary_destroy(funcs); - buffer_strcat(wb, "],\n"); + ria = qm->link.ria; + chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs); } - // context query - if (!qt->request.st) { - buffer_sprintf( - wb, - " %schart_ids%s: [", - kq, kq); + buffer_json_member_add_array(wb, key); + void *t; (void)t; + dfe_start_read(funcs, t) + buffer_json_add_array_item_string(wb, t_dfe.name); + dfe_done(t); + dictionary_destroy(funcs); + buffer_json_array_close(wb); +} - for (c = 0, i = 0; c < query_used; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; +static inline long jsonwrap_chart_labels_filter(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i = 0; + + buffer_json_member_add_object(wb, key); - if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) + SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; + char *label_key = NULL; + while (pattern && (label_key = simple_pattern_iterate(&pattern))) { + buffer_json_member_add_array(wb, label_key); + + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) continue; - if (i) - buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qm->chart.id)); - buffer_strcat(wb, sq); + QUERY_METRIC *qm = &qt->query.array[c]; + rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key); i++; } - if (!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); - } - buffer_strcat(wb, "],\n"); - if (qt->instances.chart_label_key_pattern) { - buffer_sprintf(wb, " %schart_labels%s: { ", kq, kq); - - SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; - char *label_key = NULL; - int keys = 0; - while (pattern && (label_key = simple_pattern_iterate(&pattern))) { - if (keys) - buffer_strcat(wb, ", "); - buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq); - keys++; - - for (c = 0, i = 0; c < query_used; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; - - if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) - continue; - - if (i) - buffer_strcat(wb, ", "); - rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null"); - i++; - } - if (!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); - } - buffer_strcat(wb, "]"); - } - buffer_strcat(wb, "},\n"); - } + buffer_json_array_close(wb); } - buffer_sprintf(wb, " %slatest_values%s: [" - , kq, kq); + buffer_json_object_close(wb); + + return i; +} + +static inline long jsonwrap_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; + + buffer_json_member_add_array(wb, key); for(c = 0, i = 0; c < query_used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + QUERY_METRIC *qm = &qt->query.array[c]; + buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qm->link.rma)); + i++; + } - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + buffer_json_array_close(wb); - if(i) buffer_strcat(wb, ", "); - i++; + return i; +} - NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma); - if (NAN == value) - buffer_strcat(wb, "null"); - else - buffer_rrd_value(wb, value); - } - if(!i) { - rows = 0; - buffer_strcat(wb, "null"); - } +static inline long jsonwrap_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; - buffer_sprintf(wb, "],\n" - " %sview_latest_values%s: [" - , kq, kq); + buffer_json_member_add_array(wb, key); - i = 0; - if(rows) { - NETDATA_DOUBLE total = 1; + NETDATA_DOUBLE total = 1; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - total = 0; - for(c = 0; c < query_used ;c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0; c < query_used ;c++) { + if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - NETDATA_DOUBLE n = cn[c]; + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE n = cn[c]; - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; - total += n; - } - // prevent a division by zero - if(total == 0) total = 1; + total += n; } + // prevent a division by zero + if(total == 0) total = 1; + } - for(c = 0, i = 0; c < query_used ;c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + for(c = 0, i = 0; c < query_used ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - if(i) buffer_strcat(wb, ", "); - i++; + i++; - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; - NETDATA_DOUBLE n = cn[c]; + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE n = cn[c]; - if(co[c] & RRDR_VALUE_EMPTY) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_strcat(wb, "0"); - else - buffer_strcat(wb, "null"); - } - else { - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_json_add_array_item_double(wb, 0.0); + else + buffer_json_add_array_item_double(wb, NAN); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) - n = n * 100 / total; + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; - buffer_rrd_value(wb, n); - } + buffer_json_add_array_item_double(wb, n); } } - if(!i) { - rows = 0; - buffer_strcat(wb, "null"); + + buffer_json_array_close(wb); + + return i; +} + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, + RRDR_TIME_GROUPING group_method) +{ + QUERY_TARGET *qt = r->internal.qt; + + long rows = rrdr_rows(r); + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; } - buffer_sprintf(wb, "],\n" - " %sdimensions%s: %ld,\n" - " %spoints%s: %ld,\n" - " %sformat%s: %s" - , kq, kq, i - , kq, kq, rows - , kq, kq, sq - ); + buffer_json_initialize(wb, kq, sq, 0, true); + + buffer_json_member_add_uint64(wb, "api", 1); + buffer_json_member_add_string(wb, "id", qt->id); + buffer_json_member_add_string(wb, "name", qt->id); + buffer_json_member_add_time_t(wb, "view_update_every", r->update_every); + buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); + buffer_json_member_add_time_t(wb, "after", r->after); + buffer_json_member_add_time_t(wb, "before", r->before); + buffer_json_member_add_string(wb, "group", time_grouping_tostring(group_method)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->internal.query_options); + + if(!jsonwrap_dimension_names(wb, "dimension_names", r, options)) + rows = 0; - rrdr_buffer_print_format(wb, format); + if(!jsonwrap_dimension_ids(wb, "dimension_ids", r, options)) + rows = 0; - buffer_sprintf(wb, "%s,\n" - " %sdb_points_per_tier%s: [ " - , sq - , kq, kq - ); + if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) + jsonwrap_full_dimension_list(wb, r); - for(size_t tier = 0; tier < storage_tiers ; tier++) - buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]); + jsonwrap_functions(wb, "functions", r); - buffer_strcat(wb, " ],"); + if (!qt->request.st && !jsonwrap_chart_ids(wb, "chart_ids", r, options)) + rows = 0; - if(options & RRDR_OPTION_SHOW_PLAN) - rrdr_show_plan(r, wb, kq, sq); + if (qt->instances.chart_label_key_pattern && !jsonwrap_chart_labels_filter(wb, "chart_labels", r, options)) + rows = 0; - buffer_sprintf(wb, "\n %sresult%s: ", kq, kq); + if(!jsonwrap_latest_values(wb, "latest_values", r, options)) + rows = 0; + long dimensions = jsonwrap_view_latest_values(wb, "view_latest_values", r, options); + if(!dimensions) + rows = 0; + + buffer_json_member_add_uint64(wb, "dimensions", dimensions); + buffer_json_member_add_uint64(wb, "points", rows); + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + for(size_t tier = 0; tier < storage_tiers ; tier++) + buffer_json_add_array_item_uint64(wb, r->internal.tier_points_read[tier]); + buffer_json_array_close(wb); + + if(options & RRDR_OPTION_SHOW_PLAN) + jsonwrap_query_plan(r, wb); + + buffer_sprintf(wb, ",\n %sresult%s:", kq, kq); if(string_value) buffer_strcat(wb, sq); - //info("JSONWRAPPER(): %s: END", r->st->id); } void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { @@ -455,23 +410,18 @@ void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint3 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { (void)format; - char kq[2] = "", // key quote - sq[2] = ""; // string quote + char sq[2] = ""; // string quote if( options & RRDR_OPTION_GOOGLE_JSON ) { - kq[0] = '\0'; sq[0] = '\''; } else { - kq[0] = '"'; sq[0] = '"'; } if(string_value) buffer_strcat(wb, sq); - buffer_sprintf(wb, ",\n %smin%s: ", kq, kq); - buffer_rrd_value(wb, r->min); - buffer_sprintf(wb, ",\n %smax%s: ", kq, kq); - buffer_rrd_value(wb, r->max); - buffer_strcat(wb, "\n}\n"); + buffer_json_member_add_double(wb, "min", r->min); + buffer_json_member_add_double(wb, "max", r->max); + buffer_json_finalize(wb); } |