Dyncfg add streaming support (#15791)

* dyncfg fncnames as constants

* add helper macros to know parser streaming/plugin

* plugins dictionary per RRDHOST

* api_request_v2_config add support for /host/

* streamify pluginsd_register_plugin

* streamify pluginsd_register_module

* streamify report_job_status

* streamify dyncfg get functions

* module_type2str

* add job type and flags

* add DYNCFG_REGISTER_JOB

* implement register job

* push all to parent at startup

* add helper function is_dyncfg_function

* forward virtual functions trough streaming

* separate job2json

* add api/v2/job_statuses

* do cleanup on streaming

* streamify set functions

* support FUNCTION_PAYLOAD trough streaming

* WIP tests

* dont attempt loading non-localhost configs

* move cfg persistence to proper place

* prevent race

* properly update job state at runtime

* cleanup 1

* job2json add missing reason

* add tests

* correct HTTP code

* add test

* streamify delete_job_cb

* add DELETE_JOB keyword

* job delete over streaming

* add tests for create and delete job over parent

* rrdpush common checks to macro

* add missing forwarders

* fix jobs according to test results

* more tests

* review comment 1

* codacy remove valid warning

* codacy ruby fixes

* fix wrong rc check

* minimal test plugin for child

* add test

* dict walk insted of master lock

* minor - english spelling fixes

* thiago comments 1

* minor - rename folder to dynconf

* enable only when built with -DNETDATA_TEST_DYNCFG

* minor - compiler warning

* create dir post daemonization

* stricter URL check

4 files changed, 15 insertions, 2 deletions

diff --git a/database/rrd.h b/database/rrd.h
index 20f170de1c..6c6f57c5b8 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -1254,6 +1254,8 @@ struct rrdhost {
     netdata_mutex_t aclk_state_lock;
     aclk_rrdhost_state aclk_state;
 
+    DICTIONARY *configurable_plugins;               // configurable plugins for this host
+
     struct rrdhost *next;
     struct rrdhost *prev;
 };
diff --git a/database/rrdfunctions.c b/database/rrdfunctions.c
index 1ebf12c2fa..b06d0407cc 100644
--- a/database/rrdfunctions.c
+++ b/database/rrdfunctions.c
@@ -951,12 +951,14 @@ static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool
         return rrd_call_function_async_and_dont_wait(r);
 }
 
+
+void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data);
 // ----------------------------------------------------------------------------
 
 int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
                      bool wait, const char *transaction,
                      rrd_function_result_callback_t result_cb, void *result_cb_data,
-                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data) {
+                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload) {
 
     int code;
     char sanitized_cmd[PLUGINSD_LINE_MAX + 1];
@@ -966,6 +968,12 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *
     // find the function
 
     size_t sanitized_cmd_length = sanitize_function_text(sanitized_cmd, cmd, PLUGINSD_LINE_MAX);
+
+    if (is_dyncfg_function(sanitized_cmd, DYNCFG_FUNCTION_TYPE_ALL)) {
+        call_virtual_function_async(result_wb, host, sanitized_cmd, payload, result_cb, result_cb_data);
+        return HTTP_RESP_OK;
+    }
+
     code = rrd_call_function_find(host, result_wb, sanitized_cmd, sanitized_cmd_length, &host_function_acquired);
     if(code != HTTP_RESP_OK)
         return code;
diff --git a/database/rrdfunctions.h b/database/rrdfunctions.h
index c74e4a6c01..703d3ce312 100644
--- a/database/rrdfunctions.h
+++ b/database/rrdfunctions.h
@@ -30,7 +30,7 @@ void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout,
 int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
                      bool wait, const char *transaction,
                      rrd_function_result_callback_t result_cb, void *result_cb_data,
-                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data);
+                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload);
 
 // cancel a running function, to be run from anywhere
 void rrd_function_cancel(const char *transaction);
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 2dac61de91..6abd3b8169 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -562,6 +562,9 @@ int is_legacy = 1;
          , string2str(host->health.health_default_recipient)
     );
 
+    host->configurable_plugins = dyncfg_dictionary_create();
+    dictionary_register_delete_callback(host->configurable_plugins, plugin_del_cb, NULL);
+
     if(!archived) {
         metaqueue_host_update_info(host);
         rrdhost_load_rrdcontext_data(host);