diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-02-15 21:16:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-15 21:16:29 +0200 |
commit | d2daa19bf53c9a8cb781c8e50a86b9961b0503a9 (patch) | |
tree | 8d8b744138c28e010a24456aee55447d31a719bd | |
parent | 37a918ae2bc996fc881ab60042ae5a8f434f4c52 (diff) |
JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)
* first work on standardizing json formatting
* renamed old grouping to time_grouping and added group_by
* add dummy functions to enable compilation
* buffer json api work
* jsonwrap opening with buffer_json_X() functions
* cleanup
* storage for quotes
* optimize buffer printing for both numbers and strings
* removed ; from define
* contexts json generation using the new json functions
* fix buffer overflow at unit test
* weights endpoint using new json api
* fixes to weights endpoint
* check buffer overflow on all buffer functions
* do synchronous queries for weights
* buffer_flush() now resets json state too
* content type typedef
* print double values that are above the max 64-bit value
* str2ndd() can now parse values above UINT64_MAX
* faster number parsing by avoiding double calculations as much as possible
* faster number parsing
* faster hex parsing
* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers
* full printing and parsing without using library functions - and related unit tests
* added IEEE754 streaming capability to enable streaming of double values in hex
* streaming and replication to transfer all values in hex
* use our own str2ndd for set2
* remove subnormal check from ieee
* base64 encoding for numbers, instead of hex
* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision
* str2ndd_encoded() parses all encoding formats, including integers
* prevent uninitialized use
* /api/v1/info using the new json API
* Fix error when compiling with --disable-ml
* Remove redundant 'buffer_unittest' declaration
* Fix formatting
* Fix formatting
* Fix formatting
* fix buffer unit test
* apps.plugin using the new JSON API
* make sure the metrics registry does not accept negative timestamps
* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache
* Fix more formatting
---------
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
92 files changed, 3006 insertions, 2269 deletions
diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 9eced08111..7d21a859cb 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -192,7 +192,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) w->response.data->date = w->tv_ready.tv_sec; web_client_build_http_header(w); local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk); - local_buffer->contenttype = CT_APPLICATION_JSON; + local_buffer->content_type = CT_APPLICATION_JSON; buffer_strcat(local_buffer, w->response.header_output->buffer); diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c index b432abb6e7..8f4f054dc1 100644 --- a/collectors/apps.plugin/apps_plugin.c +++ b/collectors/apps.plugin/apps_plugin.c @@ -638,9 +638,9 @@ int read_user_or_group_ids(struct user_or_group_ids *ids, struct timespec *last_ struct user_or_group_id *user_or_group_id = callocz(1, sizeof(struct user_or_group_id)); if(ids->type == USER_ID) - user_or_group_id->id.uid = (uid_t)str2ull(id_string); + user_or_group_id->id.uid = (uid_t) str2ull(id_string, NULL); else - user_or_group_id->id.gid = (uid_t)str2ull(id_string); + user_or_group_id->id.gid = (uid_t) str2ull(id_string, NULL); user_or_group_id->name = strdupz(name); user_or_group_id->updated = 1; @@ -1452,7 +1452,7 @@ static inline int read_proc_pid_stat(struct pid_stat *p, void *ptr) { pid_incremental_rate(stat, p->cstime, str2kernel_uint_t(procfile_lineword(ff, 0, 16))); // p->priority = str2kernel_uint_t(procfile_lineword(ff, 0, 17)); // p->nice = str2kernel_uint_t(procfile_lineword(ff, 0, 18)); - p->num_threads = (int32_t)str2uint32_t(procfile_lineword(ff, 0, 19)); + p->num_threads = (int32_t) str2uint32_t(procfile_lineword(ff, 0, 19), NULL); // p->itrealvalue = str2kernel_uint_t(procfile_lineword(ff, 0, 20)); p->collected_starttime = str2kernel_uint_t(procfile_lineword(ff, 0, 21)) / system_hz; p->uptime = (global_uptime > p->collected_starttime)?(global_uptime - p->collected_starttime):0; @@ -4234,7 +4234,7 @@ static void get_MemTotal(void) { for(line = 0; line < lines ;line++) { size_t words = procfile_linewords(ff, line); if(words == 3 && strcmp(procfile_lineword(ff, line, 0), "MemTotal") == 0 && strcmp(procfile_lineword(ff, line, 2), "kB") == 0) { - kernel_uint_t n = str2ull(procfile_lineword(ff, line, 1)); + kernel_uint_t n = str2ull(procfile_lineword(ff, line, 1), NULL); if(n) MemTotal = n; break; } @@ -4289,47 +4289,41 @@ static void apps_plugin_function_processes_help(const char *transaction) { } #define add_table_field(wb, key, name, visible, type, visualization, transform, decimal_points, units, max, sort, sortable, sticky, unique_key, pointer_to, summary, range) do { \ - if(fields_added) buffer_strcat(wb, ","); \ - buffer_sprintf(wb, "\n \"%s\": {", key); \ - buffer_sprintf(wb, "\n \"index\":%d,", fields_added); \ - buffer_sprintf(wb, "\n \"unique_key\":%s,", (unique_key)?"true":"false"); \ - buffer_sprintf(wb, "\n \"name\":\"%s\",", name); \ - buffer_sprintf(wb, "\n \"visible\":%s,", (visible)?"true":"false"); \ - buffer_sprintf(wb, "\n \"type\":\"%s\",", type); \ - if(units) \ - buffer_sprintf(wb, "\n \"units\":\"%s\",", (char*)(units)); \ - buffer_sprintf(wb, "\n \"visualization\":\"%s\",", visualization); \ - buffer_sprintf(wb, "\n \"value_options\":{"); \ - if(units) \ - buffer_sprintf(wb, "\n \"units\":\"%s\",", (char*)(units)); \ - buffer_sprintf(wb, "\n \"transform\":\"%s\",", transform); \ - buffer_sprintf(wb, "\n \"decimal_points\":%d", decimal_points); \ - buffer_sprintf(wb, "\n },"); \ + buffer_json_member_add_object(wb, key); \ + buffer_json_member_add_uint64(wb, "index", fields_added); \ + buffer_json_member_add_boolean(wb, "unique_key", unique_key); \ + buffer_json_member_add_string(wb, "name", name); \ + buffer_json_member_add_boolean(wb, "visible", visible); \ + buffer_json_member_add_string(wb, "type", type); \ + buffer_json_member_add_string_or_omit(wb, "units", (char*)(units)); \ + buffer_json_member_add_string(wb, "visualization", visualization); \ + buffer_json_member_add_object(wb, "value_options"); \ + buffer_json_member_add_string_or_omit(wb, "units", (char*)(units)); \ + buffer_json_member_add_string(wb, "transform", transform); \ + buffer_json_member_add_uint64(wb, "decimal_points", decimal_points); \ + buffer_json_object_close(wb); \ if(!isnan((NETDATA_DOUBLE)(max))) \ - buffer_sprintf(wb, "\n \"max\":%f,", (NETDATA_DOUBLE)(max)); \ - if(pointer_to) \ - buffer_sprintf(wb, "\n \"pointer_to\":\"%s\",", (char *)(pointer_to)); \ - buffer_sprintf(wb, "\n \"sort\":\"%s\",", sort); \ - buffer_sprintf(wb, "\n \"sortable\":%s,", (sortable)?"true":"false"); \ - buffer_sprintf(wb, "\n \"sticky\":%s,", (sticky)?"true":"false"); \ - buffer_sprintf(wb, "\n \"summary\":\"%s\",", summary); \ - buffer_sprintf(wb, "\n \"filter\":\"%s\"", (range)?"range":"multiselect"); \ - buffer_sprintf(wb, "\n }"); \ + buffer_json_member_add_double(wb, "max", (NETDATA_DOUBLE)(max)); \ + buffer_json_member_add_string_or_omit(wb, "pointer_to", (char *)(pointer_to)); \ + buffer_json_member_add_string(wb, "sort", sort); \ + buffer_json_member_add_boolean(wb, "sortable", sortable); \ + buffer_json_member_add_boolean(wb, "sticky", sticky); \ + buffer_json_member_add_string(wb, "summary", summary); \ + buffer_json_member_add_string(wb, "filter", (range)?"range":"multiselect"); \ + buffer_json_object_close(wb); \ fields_added++; \ } while(0) #define add_value_field_llu_with_max(wb, key, value) do { \ unsigned long long _tmp = (value); \ key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp); \ - buffer_fast_strcat(wb, ",", 1); \ - buffer_print_llu(wb, _tmp); \ + buffer_json_add_array_item_uint64(wb, _tmp); \ } while(0) #define add_value_field_ndd_with_max(wb, key, value) do { \ NETDATA_DOUBLE _tmp = (value); \ key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp); \ - buffer_fast_strcat(wb, ",", 1); \ - buffer_rrd_value(wb, _tmp); \ + buffer_json_add_array_item_double(wb, _tmp); \ } while(0) static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) { @@ -4406,18 +4400,12 @@ static void apps_plugin_function_processes(const char *transaction, char *functi unsigned int io_divisor = 1024 * RATES_DETAIL; BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); - buffer_sprintf(wb, - "{" - "\n \"status\":%d" - ",\n \"type\":\"table\"" - ",\n \"update_every\":%d" - ",\n \"help\":\"%s\"" - ",\n \"data\":[" - "\n" - , HTTP_RESP_OK - , update_every - , APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION - ); + buffer_json_initialize(wb, "\"", "\"", 0, true); + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); + buffer_json_member_add_string(wb, "type", "table"); + buffer_json_member_add_time_t(wb, "update_every", update_every); + buffer_json_member_add_string(wb, "help", APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); + buffer_json_member_add_array(wb, "data"); NETDATA_DOUBLE UserCPU_max = 0.0 @@ -4493,52 +4481,41 @@ static void apps_plugin_function_processes(const char *transaction, char *functi |