diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-12-15 18:15:43 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-12-15 18:15:43 +0200 |
commit | da32dd8be88ec9e88921d134b02df8ab85856779 (patch) | |
tree | 09e44dd48f6641fb0f6de9064bbea572572162c8 /web/api/web_api_v2.c | |
parent | 0c8b46cbfd05109a45ee4de27f034567569fa3fa (diff) |
Queries Progress (#16574)
* track the progress of queries
* add query_progress in libnetdata Makefile.am
* add acl, response size and response code to the tracking
* define the required functions
* fix the last commit
* added /api/v2/progress?transaction=ID to report the progress of queries
* added function to report netdata-queries
* track hashtable additions
* when resusing a transaction, maintain the counter
* keep track of linked and indexing
* added X-Forwarded-Host and X-Forwarded-For to logs. X-Forwarded-For is also added in progress tracking
* report compact uuids to match logs; register the actual duration of the transaction
* added rowOptions to function; now web_client keeps track if it tracks progress or not
* add http request method to progress
* add tags per function; /api/vX/functions is now not protected
* compact the sanitization array
* split pluginsd_parser into multiple files
* cleanup keyword definitions
* code cleanup
* extracted rrd_collector to separate files
* added http access level to functions
* renamed access "all" to "any"
* implemented optional protection on functions
* add priority to functions, to allow the UI select the best function (lower priority) when the user has not selected a function
* added progress report from the plugins to netdata and from children to parents - untested
* added progress reporting in systemd-journal
* query timeout is now handled by evloop for external plugins
* propagate progress reports to children and plugins
* fix codeql warning
* adapt to cmake
* minor changes
* extend function timeout when progress is received; added streaming capability to propagate progress reports to parents and send progress requests to children
* revert change in dictionary.h
* add log when access level is invalid
* update access level of functions
* added logs when processing progress updates
* log when the deferred response is too big
* comment out sender progress to find the issue
* added missing newline in streaming progress reports
* propogate progress reports to functions
* fix logs
Diffstat (limited to 'web/api/web_api_v2.c')
-rw-r--r-- | web/api/web_api_v2.c | 84 |
1 files changed, 53 insertions, 31 deletions
diff --git a/web/api/web_api_v2.c b/web/api/web_api_v2.c index 9daf80b9d2..5937fc4cb7 100644 --- a/web/api/web_api_v2.c +++ b/web/api/web_api_v2.c @@ -580,6 +580,8 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w .interrupt_callback = web_client_interrupt_callback, .interrupt_callback_data = w, + + .transaction = &w->transaction, }; for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) @@ -662,6 +664,31 @@ static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct return webrtc_new_connection(w->post_payload, w->response.data); } +static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, struct web_client *w, char *url) { + char *transaction = NULL; + + while(url) { + char *value = strsep_skip_consecutive_separators(&url, "&"); + if(!value || !*value) continue; + + char *name = strsep_skip_consecutive_separators(&value, "="); + if(!name || !*name) continue; + if(!value || !*value) continue; + + // name and value are now the parameters + // they are not null and not empty + + if(!strcmp(name, "transaction")) transaction = value; + } + + uuid_t tr; + uuid_parse_flexi(transaction, tr); + + rrd_function_call_progresser(&tr); + + return web_api_v2_report_progress(&tr, w->response.data); +} + #define CONFIG_API_V2_URL "/api/v2/config" static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct web_client *w, char *query __maybe_unused) { @@ -703,17 +730,11 @@ static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct int http_method; switch (w->mode) { - case WEB_CLIENT_MODE_GET: - http_method = HTTP_METHOD_GET; - break; - case WEB_CLIENT_MODE_POST: - http_method = HTTP_METHOD_POST; - break; - case WEB_CLIENT_MODE_PUT: - http_method = HTTP_METHOD_PUT; - break; - case WEB_CLIENT_MODE_DELETE: - http_method = HTTP_METHOD_DELETE; + case HTTP_REQUEST_MODE_GET: + case HTTP_REQUEST_MODE_POST: + case HTTP_REQUEST_MODE_PUT: + case HTTP_REQUEST_MODE_DELETE: + http_method = w->mode; break; default: buffer_sprintf(w->response.data, "Invalid HTTP method"); @@ -839,35 +860,36 @@ static int web_client_api_request_v2_job_statuses(RRDHOST *host __maybe_unused, } static struct web_api_command api_commands_v2[] = { - {"info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0}, + {"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0}, - {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0}, - {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0}, + {"data", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0}, + {"weights", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0}, - {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0}, - {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0}, - {"node_instances", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0}, - {"versions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0}, - {"functions", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0}, - {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0}, - {"alerts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0}, + {"contexts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0}, + {"nodes", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0}, + {"node_instances", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0}, + {"versions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0}, + {"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0}, + {"q", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0}, + {"alerts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0}, - {"alert_transitions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0}, - {"alert_config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0}, + {"alert_transitions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0}, + {"alert_config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0}, - {"claim", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_claim, 0}, + {"claim", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_claim, 0}, - {"rtc_offer", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0}, - {"bearer_protection", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0}, - {"bearer_get_token", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0}, + {"rtc_offer", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0}, + {"bearer_protection", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0}, + {"bearer_get_token", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0}, - {"config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1}, - {"job_statuses", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0}, + {"config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1}, + {"job_statuses", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0}, - { "ilove.svg", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 }, + { "ilove.svg", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 }, + { "progress", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_progress, 0 }, // terminator - {NULL, 0, WEB_CLIENT_ACL_NONE, NULL, 0}, + {NULL, 0, HTTP_ACL_NONE, NULL, 0}, }; inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) { |