Queries Progress (#16574)

* track the progress of queries

* add query_progress in libnetdata Makefile.am

* add acl, response size and response code to the tracking

* define the required functions

* fix the last commit

* added /api/v2/progress?transaction=ID to report the progress of queries

* added function to report netdata-queries

* track hashtable additions

* when resusing a transaction, maintain the counter

* keep track of linked and indexing

* added X-Forwarded-Host and X-Forwarded-For to logs. X-Forwarded-For is also added in progress tracking

* report compact uuids to match logs; register the actual duration of the transaction

* added rowOptions to function; now web_client keeps track if it tracks progress or not

* add http request method to progress

* add tags per function; /api/vX/functions is now not protected

* compact the sanitization array

* split pluginsd_parser into multiple files

* cleanup keyword definitions

* code cleanup

* extracted rrd_collector to separate files

* added http access level to functions

* renamed access "all" to "any"

* implemented optional protection on functions

* add priority to functions, to allow the UI select the best function (lower priority) when the user has not selected a function

* added progress report from the plugins to netdata and from children to parents - untested

* added progress reporting in systemd-journal

* query timeout is now handled by evloop for external plugins

* propagate progress reports to children and plugins

* fix codeql warning

* adapt to cmake

* minor changes

* extend function timeout when progress is received; added streaming capability to propagate progress reports to parents and send progress requests to children

* revert change in dictionary.h

* add log when access level is invalid

* update access level of functions

* added logs when processing progress updates

* log when the deferred response is too big

* comment out sender progress to find the issue

* added missing newline in streaming progress reports

* propogate progress reports to functions

* fix logs

1 files changed, 13 insertions, 7 deletions

diff --git a/collectors/proc.plugin/proc_diskstats.c b/collectors/proc.plugin/proc_diskstats.c
index 475d90835f..dd35d70401 100644
--- a/collectors/proc.plugin/proc_diskstats.c
+++ b/collectors/proc.plugin/proc_diskstats.c
@@ -1033,12 +1033,16 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) {
     rrdlabels_add(st->rrdlabels, "device_type", get_disk_type_string(d->type), RRDLABEL_SRC_AUTO);
 }
 
-static int diskstats_function_block_devices(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
-        void *collector_data __maybe_unused,
-        rrd_function_result_callback_t result_cb, void *result_cb_data,
-        rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
-        rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
-        void *register_canceller_cb_data __maybe_unused) {
+static int diskstats_function_block_devices(uuid_t *transaction __maybe_unused, BUFFER *wb,
+                                            usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
+                                            void *collector_data __maybe_unused,
+                                            rrd_function_result_callback_t result_cb, void *result_cb_data,
+                                            rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
+                                            rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
+                                            rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
+                                            void *register_canceller_cb_data __maybe_unused,
+                                            rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
+                                            void *register_progresser_cb_data __maybe_unused) {
 
     buffer_flush(wb);
     wb->content_type = CT_APPLICATION_JSON;
@@ -1490,7 +1494,9 @@ int do_proc_diskstats(int update_every, usec_t dt) {
 
     static bool add_func = true;
     if (add_func) {
-        rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_DISKSTATS_HELP, true, diskstats_function_block_devices, NULL);
+        rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSTATS_HELP,
+                         "top", HTTP_ACCESS_ANY, true,
+                         diskstats_function_block_devices, NULL);
         add_func = false;
     }