diff options
author | vkalintiris <vasilis@netdata.cloud> | 2023-01-04 14:51:25 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-04 14:51:25 +0200 |
commit | 78359cd375d0b2c285741e6f934a681d0a0c3c15 (patch) | |
tree | 2d5264325510b663d9e87ca62a38fad187e3a713 /ml/ml.cc | |
parent | df379e45fbaddf825f1f7972a75ae3f3daf80097 (diff) |
Refactor ML code and add support for multiple KMeans models (#14198)
* Add profile.plugin
Creates the specified number of charts/dimensions, and supports
backfilling with pseudo-historical data.
* Bump
* Remove wrongly merged line.
* Use the number of models specified from the config section.
* Add option to consult all ML models.
* Remove profiling option consuming all models.
* Add underscore after chart name prefix.
* prediction -> dimensions chart
* reorder funcs
* Split charts across types with correct priority
* Ignore training request when chart is under replication.
* Track global number of models consulted.
* Cleanup config.
* initial readme updates
* fix readme
* readme
* Fix function definition when ML is disabled.
* Add dummy ml_chart_update_{begin,end}
* Remove profile_plugin
* Define chart priorities under collectors/all.h
* s/curr_t/current_time/
* Use libnetdata's lock/thread wrappers.
* Fix autotools & cmake builds.
* Delete ML dimensions & charts.
* Let users of buffer preprocessing to handle memory.
* Add separate API calls to start/stop ML threads.
Co-authored-by: Andrew Maguire <andrewm4894@gmail.com>
Diffstat (limited to 'ml/ml.cc')
-rw-r--r-- | ml/ml.cc | 108 |
1 files changed, 71 insertions, 37 deletions
@@ -2,6 +2,7 @@ #include "Config.h" #include "Dimension.h" +#include "Chart.h" #include "Host.h" #include <random> @@ -45,18 +46,18 @@ void ml_init(void) { Cfg.RandomNums.push_back(Gen()); } -void ml_new_host(RRDHOST *RH) { +void ml_host_new(RRDHOST *RH) { if (!ml_enabled(RH)) return; Host *H = new Host(RH); - RH->ml_host = static_cast<ml_host_t>(H); + RH->ml_host = reinterpret_cast<ml_host_t *>(H); H->startAnomalyDetectionThreads(); } -void ml_delete_host(RRDHOST *RH) { - Host *H = static_cast<Host *>(RH->ml_host); +void ml_host_delete(RRDHOST *RH) { + Host *H = reinterpret_cast<Host *>(RH->ml_host); if (!H) return; @@ -66,35 +67,48 @@ void ml_delete_host(RRDHOST *RH) { RH->ml_host = nullptr; } -void ml_new_dimension(RRDDIM *RD) { - RRDSET *RS = RD->rrdset; - - Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host); +void ml_chart_new(RRDSET *RS) { + Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host); if (!H) return; - if (static_cast<unsigned>(RD->update_every) != H->updateEvery()) + Chart *C = new Chart(RS); + RS->ml_chart = reinterpret_cast<ml_chart_t *>(C); + + H->addChart(C); +} + +void ml_chart_delete(RRDSET *RS) { + Host *H = reinterpret_cast<Host *>(RS->rrdhost->ml_host); + if (!H) return; - if (simple_pattern_matches(Cfg.SP_ChartsToSkip, rrdset_name(RS))) + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart); + H->removeChart(C); + + delete C; + RS->ml_chart = nullptr; +} + +void ml_dimension_new(RRDDIM *RD) { + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart); + if (!C) return; Dimension *D = new Dimension(RD); - RD->ml_dimension = static_cast<ml_dimension_t>(D); - H->addDimension(D); + RD->ml_dimension = reinterpret_cast<ml_dimension_t *>(D); + C->addDimension(D); } -void ml_delete_dimension(RRDDIM *RD) { - Dimension *D = static_cast<Dimension *>(RD->ml_dimension); +void ml_dimension_delete(RRDDIM *RD) { + Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension); if (!D) return; - Host *H = static_cast<Host *>(RD->rrdset->rrdhost->ml_host); - if (!H) - delete D; - else - H->removeDimension(D); + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart); + C->removeDimension(D); + delete D; RD->ml_dimension = nullptr; } @@ -102,7 +116,7 @@ char *ml_get_host_info(RRDHOST *RH) { nlohmann::json ConfigJson; if (RH && RH->ml_host) { - Host *H = static_cast<Host *>(RH->ml_host); + Host *H = reinterpret_cast<Host *>(RH->ml_host); H->getConfigAsJson(ConfigJson); } else { ConfigJson["enabled"] = false; @@ -115,7 +129,7 @@ char *ml_get_host_runtime_info(RRDHOST *RH) { nlohmann::json ConfigJson; if (RH && RH->ml_host) { - Host *H = static_cast<Host *>(RH->ml_host); + Host *H = reinterpret_cast<Host *>(RH->ml_host); H->getDetectionInfoAsJson(ConfigJson); } else { return nullptr; @@ -128,7 +142,7 @@ char *ml_get_host_models(RRDHOST *RH) { nlohmann::json ModelsJson; if (RH && RH->ml_host) { - Host *H = static_cast<Host *>(RH->ml_host); + Host *H = reinterpret_cast<Host *>(RH->ml_host); H->getModelsAsJson(ModelsJson); return strdup(ModelsJson.dump(2, '\t').c_str()); } @@ -136,30 +150,50 @@ char *ml_get_host_models(RRDHOST *RH) { return nullptr; } -bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) { - Dimension *D = static_cast<Dimension *>(RD->ml_dimension); - if (!D) - return false; +void ml_start_anomaly_detection_threads(RRDHOST *RH) { + if (RH && RH->ml_host) { + Host *H = reinterpret_cast<Host *>(RH->ml_host); + H->startAnomalyDetectionThreads(); + } +} - return D->predict(Value, Exists); +void ml_stop_anomaly_detection_threads(RRDHOST *RH) { + if (RH && RH->ml_host) { + Host *H = reinterpret_cast<Host *>(RH->ml_host); + H->stopAnomalyDetectionThreads(); + } } -bool ml_streaming_enabled() { - return Cfg.StreamADCharts; +void ml_chart_update_begin(RRDSET *RS) { + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart); + if (!C) + return; + + C->updateBegin(); } -#if defined(ENABLE_ML_TESTS) +void ml_chart_update_end(RRDSET *RS) { + Chart *C = reinterpret_cast<Chart *>(RS->ml_chart); + if (!C) + return; + + C->updateEnd(); +} -#include "gtest/gtest.h" +bool ml_is_anomalous(RRDDIM *RD, time_t CurrT, double Value, bool Exists) { + Dimension *D = reinterpret_cast<Dimension *>(RD->ml_dimension); + if (!D) + return false; -int test_ml(int argc, char *argv[]) { - (void) argc; - (void) argv; + Chart *C = reinterpret_cast<Chart *>(RD->rrdset->ml_chart); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + bool IsAnomalous = D->predict(CurrT, Value, Exists); + C->updateDimension(D, IsAnomalous); + return IsAnomalous; } -#endif // ENABLE_ML_TESTS +bool ml_streaming_enabled() { + return Cfg.StreamADCharts; +} #include "ml-private.h" |