diff options
author | vkalintiris <vasilis@netdata.cloud> | 2021-12-22 17:13:45 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-12-22 17:13:45 +0200 |
commit | 5049de8f7a132205b32f2c523998f93cbbfc0481 (patch) | |
tree | 452a81eadda2ac3354bc5ceff0d529e47520c578 | |
parent | df8930ddd370b2a9fec96b1cbb45e8ae530c0aad (diff) |
Provide runtime ml info from a new endpoint. (#11886)
* Provide runtime ml info from a new endpoint.
* Add hosts & charts to skip from ML in the /info endpoint.
This information belongs in /info, and not in /ml-info, because the
value of these variables can not change at the agent's runtime.
* Use strdupz instead of strdup.
-rw-r--r-- | ml/Config.cc | 14 | ||||
-rw-r--r-- | ml/Config.h | 3 | ||||
-rw-r--r-- | ml/Host.cc | 4 | ||||
-rw-r--r-- | ml/ml.cc | 14 | ||||
-rw-r--r-- | ml/ml.h | 1 | ||||
-rw-r--r-- | web/api/web_api_v1.c | 30 |
6 files changed, 54 insertions, 12 deletions
diff --git a/ml/Config.cc b/ml/Config.cc index f48f9b39fb..df0a2825e3 100644 --- a/ml/Config.cc +++ b/ml/Config.cc @@ -47,12 +47,6 @@ void Config::readMLConfig(void) { double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25); double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05); - std::string HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*"); - std::string ChartsToSkip = config_get(ConfigSectionML, "charts to skip from training", - "!system.* !cpu.* !mem.* !disk.* !disk_* " - "!ip.* !ipv4.* !ipv6.* !net.* !net_* !netfilter.* " - "!services.* !apps.* !groups.* !user.* !ebpf.* !netdata.* *"); - std::stringstream SS; SS << netdata_configured_cache_dir << "/anomaly-detection.db"; Cfg.AnomalyDBPath = SS.str(); @@ -123,6 +117,12 @@ void Config::readMLConfig(void) { Cfg.ADWindowRateThreshold = ADWindowRateThreshold; Cfg.ADDimensionRateThreshold = ADDimensionRateThreshold; - Cfg.SP_HostsToSkip = simple_pattern_create(HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT); + Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*"); + Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT); + + Cfg.ChartsToSkip = config_get(ConfigSectionML, "charts to skip from training", + "!system.* !cpu.* !mem.* !disk.* !disk_* " + "!ip.* !ipv4.* !ipv6.* !net.* !net_* !netfilter.* " + "!services.* !apps.* !groups.* !user.* !ebpf.* !netdata.* *"); Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT); } diff --git a/ml/Config.h b/ml/Config.h index f29bae3a60..58ecce53f3 100644 --- a/ml/Config.h +++ b/ml/Config.h @@ -30,7 +30,10 @@ public: double ADWindowRateThreshold; double ADDimensionRateThreshold; + std::string HostsToSkip; SIMPLE_PATTERN *SP_HostsToSkip; + + std::string ChartsToSkip; SIMPLE_PATTERN *SP_ChartsToSkip; std::string AnomalyDBPath; diff --git a/ml/Host.cc b/ml/Host.cc index d26ff2ae40..b632710a44 100644 --- a/ml/Host.cc +++ b/ml/Host.cc @@ -282,6 +282,9 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const { Json["idle-window-size"] = Cfg.ADIdleWindowSize; Json["window-rate-threshold"] = Cfg.ADWindowRateThreshold; Json["dimension-rate-threshold"] = Cfg.ADDimensionRateThreshold; + + Json["hosts-to-skip"] = Cfg.HostsToSkip; + Json["charts-to-skip"] = Cfg.ChartsToSkip; } std::pair<Dimension *, Duration<double>> @@ -441,6 +444,7 @@ void DetectableHost::detect() { } void DetectableHost::getDetectionInfoAsJson(nlohmann::json &Json) const { + Json["version"] = 1; Json["anomalous-dimensions"] = NumAnomalousDimensions; Json["normal-dimensions"] = NumNormalDimensions; Json["total-dimensions"] = NumAnomalousDimensions + NumNormalDimensions; @@ -75,7 +75,6 @@ char *ml_get_host_info(RRDHOST *RH) { if (RH && RH->ml_host) { Host *H = static_cast<Host *>(RH->ml_host); H->getConfigAsJson(ConfigJson); - H->getDetectionInfoAsJson(ConfigJson); } else { ConfigJson["enabled"] = false; } @@ -83,6 +82,19 @@ char *ml_get_host_info(RRDHOST *RH) { return strdup(ConfigJson.dump(2, '\t').c_str()); } +char *ml_get_host_runtime_info(RRDHOST *RH) { + nlohmann::json ConfigJson; + + if (RH && RH->ml_host) { + Host *H = static_cast<Host *>(RH->ml_host); + H->getDetectionInfoAsJson(ConfigJson); + } else { + return nullptr; + } + + return strdup(ConfigJson.dump(1, '\t').c_str()); +} + bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) { Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension); if (!D) @@ -18,6 +18,7 @@ void ml_new_host(RRDHOST *RH); void ml_delete_host(RRDHOST *RH); char *ml_get_host_info(RRDHOST *RH); +char *ml_get_host_runtime_info(RRDHOST *RH); void ml_new_dimension(RRDDIM *RD); void ml_delete_dimension(RRDDIM *RD); diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index d335dd687d..ff5af93b32 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -1130,11 +1130,11 @@ int web_client_api_request_v1_anomaly_events(RRDHOST *host, struct web_client *w char *s; if (!before || !after) - s = strdup("{\"error\": \"missing after/before parameters\" }\n"); + s = strdupz("{\"error\": \"missing after/before parameters\" }\n"); else { s = ml_get_anomaly_events(host, "AD1", 1, after, before); if (!s) - s = strdup("{\"error\": \"json string is empty\" }\n"); + s = strdupz("{\"error\": \"json string is empty\" }\n"); } BUFFER *wb = w->response.data; @@ -1174,11 +1174,11 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien char *s; if (!before || !after) - s = strdup("{\"error\": \"missing after/before parameters\" }\n"); + s = strdupz("{\"error\": \"missing after/before parameters\" }\n"); else { s = ml_get_anomaly_event_info(host, "AD1", 1, after, before); if (!s) - s = strdup("{\"error\": \"json string is empty\" }\n"); + s = strdupz("{\"error\": \"json string is empty\" }\n"); } BUFFER *wb = w->response.data; @@ -1190,6 +1190,27 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien freez(s); return HTTP_RESP_OK; } + +int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) { + (void) url; + + if (!netdata_ready) + return HTTP_RESP_BACKEND_FETCH_FAILED; + + char *s = ml_get_host_runtime_info(host); + if (!s) + s = strdupz("{\"error\": \"json string is empty\" }\n"); + + BUFFER *wb = w->response.data; + buffer_flush(wb); + wb->contenttype = CT_APPLICATION_JSON; + buffer_strcat(wb, s); + buffer_no_cacheable(wb); + + freez(s); + return HTTP_RESP_OK; +} + #endif // defined(ENABLE_ML) inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url) { @@ -1250,6 +1271,7 @@ static struct api_command { #if defined(ENABLE_ML) { "anomaly_events", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_events }, { "anomaly_event_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_event_info }, + { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info }, #endif { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health }, |