Remove anomaly detector (#13657)

* Move all dims under one class.

* Dimension owns anomaly rate RD.

* Remove Dimension::isAnomalous()

* Remove Dimension::trainEvery()

* Rm ml/kmeans

* Remove anomaly detector

The same logic can be implemented by using the host anomaly rate dim.

* Profile plugin.

* Revert "Profile plugin."

This reverts commit e3db37cb49c514502c5216cfe7bca2a003fb90f1.

* Add separate source files for anomaly detection charts.

* Handle training/prediction sync at the dimension level.

* Keep multiple KMeans models in mem.

* Move feature extraction outside KMeans class.

* Use multiple models.

* Add /api/v1/ml_models endpoint.

* Remove Dimension::getID()

* Use just 1 model and fix tests.

* Add detection logic based on rrdr.

* Remove config options related to anomaly detection.

* Make anomaly detection queries configurable.

* Fix ad query duration option.

* Finalize queries in all code paths.

* Check if query was initialized before finalizing it

* Do not leak OWA

* Profile plugin.

* Revert "Profile plugin."

This reverts commit 5c77145d0df7e091d030476c480ab8d9cbceb89e.

* Change context from anomaly_detection to detector_events.

29 files changed, 535 insertions, 1546 deletions

diff --git a/ml/ADCharts.cc b/ml/ADCharts.cc
new file mode 100644
index 0000000000..5a78b97918
--- /dev/null
+++ b/ml/ADCharts.cc
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "ADCharts.h"
+#include "Config.h"
+
+void ml::updateDimensionsChart(RRDHOST *RH,
+                               collected_number NumTrainedDimensions,
+                               collected_number NumNormalDimensions,
+                               collected_number NumAnomalousDimensions) {
+    static thread_local RRDSET *RS = nullptr;
+    static thread_local RRDDIM *NumTotalDimensionsRD = nullptr;
+    static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr;
+    static thread_local RRDDIM *NumNormalDimensionsRD = nullptr;
+    static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr;
+
+    if (!RS) {
+        std::stringstream IdSS, NameSS;
+
+        IdSS << "dimensions_on_" << localhost->machine_guid;
+        NameSS << "dimensions_on_" << localhost->hostname;
+
+        RS = rrdset_create(
+            RH,
+            "anomaly_detection", // type
+            IdSS.str().c_str(), // id
+            NameSS.str().c_str(), // name
+            "dimensions", // family
+            "anomaly_detection.dimensions", // ctx
+            "Anomaly detection dimensions", // title
+            "dimensions", // units
+            "netdata", // plugin
+            "ml", // module
+            39183, // priority
+            RH->rrd_update_every, // update_every
+            RRDSET_TYPE_LINE // chart_type
+        );
+        rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION);
+
+        NumTotalDimensionsRD = rrddim_add(RS, "total", NULL,
+                1, 1, RRD_ALGORITHM_ABSOLUTE);
+        NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL,
+                1, 1, RRD_ALGORITHM_ABSOLUTE);
+        NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL,
+                1, 1, RRD_ALGORITHM_ABSOLUTE);
+        NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL,
+                1, 1, RRD_ALGORITHM_ABSOLUTE);
+    } else
+        rrdset_next(RS);
+
+    rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions);
+    rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions);
+    rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions);
+    rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions);
+
+    rrdset_done(RS);
+}
+
+void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) {
+    static thread_local RRDSET *HostRateRS = nullptr;
+    static thread_local RRDDIM *AnomalyRateRD = nullptr;
+
+    if (!HostRateRS) {
+        std::stringstream IdSS, NameSS;
+
+        IdSS << "anomaly_rate_on_" << localhost->machine_guid;
+        NameSS << "anomaly_rate_on_" << localhost->hostname;
+
+        HostRateRS = rrdset_create(
+            RH,
+            "anomaly_detection", // type
+            IdSS.str().c_str(), // id
+            NameSS.str().c_str(), // name
+            "anomaly_rate", // family
+            "anomaly_detection.anomaly_rate", // ctx
+            "Percentage of anomalous dimensions", // title
+            "percentage", // units
+            "netdata", // plugin
+            "ml", // module
+            39184, // priority
+            RH->rrd_update_every, // update_every
+            RRDSET_TYPE_LINE // chart_type
+        );
+        rrdset_flag_set(HostRateRS, RRDSET_FLAG_ANOMALY_DETECTION);
+
+        AnomalyRateRD = rrddim_add(HostRateRS, "anomaly_rate", NULL,
+                1, 100, RRD_ALGORITHM_ABSOLUTE);
+    } else
+        rrdset_next(HostRateRS);
+
+    rrddim_set_by_pointer(HostRateRS, AnomalyRateRD, AnomalyRate);
+    rrdset_done(HostRateRS);
+
+    static thread_local RRDSET *AnomalyDetectionRS = nullptr;
+    static thread_local RRDDIM *AboveThresholdRD = nullptr;
+    static thread_local RRDDIM *NewAnomalyEventRD = nullptr;
+
+    if (!AnomalyDetectionRS) {
+        std::stringstream IdSS, NameSS;
+
+        IdSS << "anomaly_detection_on_" << localhost->machine_guid;
+        NameSS << "anomaly_detection_on_" << localhost->hostname;
+
+        AnomalyDetectionRS = rrdset_create(
+            RH,
+            "anomaly_detection", // type
+            IdSS.str().c_str(), // id
+            NameSS.str().c_str(), // name
+            "anomaly_detection", // family
+            "anomaly_detection.detector_events", // ctx
+            "Anomaly detection events", // title
+            "percentage", // units
+            "netdata", // plugin
+            "ml", // module
+            39185, // priority
+            RH->rrd_update_every, // update_every
+            RRDSET_TYPE_LINE // chart_type
+        );
+        rrdset_flag_set(AnomalyDetectionRS, RRDSET_FLAG_ANOMALY_DETECTION);
+
+        AboveThresholdRD  = rrddim_add(AnomalyDetectionRS, "above_threshold", NULL,
+                                       1, 1, RRD_ALGORITHM_ABSOLUTE);
+        NewAnomalyEventRD = rrddim_add(AnomalyDetectionRS, "new_anomaly_event", NULL,
+                                       1, 1, RRD_ALGORITHM_ABSOLUTE);
+    } else
+        rrdset_next(AnomalyDetectionRS);
+
+    /*
+     * Compute the values of the dimensions based on the host rate chart
+    */
+    ONEWAYALLOC *OWA = onewayalloc_create(0);
+    time_t Now = now_realtime_sec();
+    time_t Before = Now - RH->rrd_update_every;
+    time_t After = Before - Cfg.AnomalyDetectionQueryDuration;
+    RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000);
+
+    RRDR *R = rrd2rrdr(
+        OWA, HostRateRS,
+        1 /* points wanted */,
+        After,
+        Before,
+        Cfg.AnomalyDetectionGroupingMethod,
+        0 /* resampling time */,
+        Options, "anomaly_rate",
+        NULL /* context param list */,
+        NULL /* group options */,
+        0, /* timeout */
+        0 /* tier */
+    );
+    assert(R->d == 1 && R->n == 1 && R->rows == 1);
+
+    static thread_local bool PrevAboveThreshold = false;
+    bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold;
+    bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold;
+    PrevAboveThreshold = AboveThreshold;
+
+    rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold);
+    rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent);
+    rrdset_done(AnomalyDetectionRS);
+
+    rrdr_free(OWA, R);
+    onewayalloc_destroy(OWA);
+}
+
+void ml::updateDetectionChart(RRDHOST *RH) {
+    static thread_local RRDSET *RS = nullptr;
+    static thread_local RRDDIM *UserRD, *SystemRD = nullptr;
+
+    if (!RS) {
+        std::stringstream IdSS, NameSS;
+
+        IdSS << "prediction_stats_" << RH->machine_guid;
+        NameSS << "prediction_stats_for_" << RH->hostname;
+
+        RS = rrdset_create_localhost(
+            "netdata", // type
+            IdSS.str().c_str(), // id
+            NameSS.str().c_str(), // name
+            "ml", // family
+            "netdata.prediction_stats", // ctx
+            "Prediction thread CPU usage", // title
+            "milliseconds/s", // units
+            "netdata", // plugin
+            "ml", // module
+            136000, // priority
+            RH->rrd_update_every, // update_every
+            RRDSET_TYPE_STACKED // chart_type
+        );
+
+        UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+        SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+    } else
+        rrdset_next(RS);
+
+    struct rusage TRU;
+    getrusage(RUSAGE_THREAD, &TRU);
+
+    rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec);
+    rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec);
+    rrdset_done(RS);
+}
+
+void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) {
+    static thread_local RRDSET *RS = nullptr;
+    static thread_local RRDDIM *UserRD = nullptr;
+    static thread_local RRDDIM *SystemRD = nullptr;
+
+    if (!RS) {
+        std::stringstream IdSS, NameSS;
+
+        IdSS << "training_stats_" << RH->machine_guid;
+        NameSS << "training_stats_for_" << RH->hostname;
+
+        RS = rrdset_create_localhost(
+            "netdata", // type
+            IdSS.str().c_str(), // id
+            NameSS.str().c_str(), // name
+            "ml", // family
+            "netdata.training_stats", // ctx
+            "Training thread CPU usage", // title
+            "milliseconds/s", // units
+            "netdata", // plugin
+            "ml", // module
+            136001, // priority
+            RH->rrd_update_every, // update_every
+            RRDSET_TYPE_STACKED // chart_type
+        );
+
+        UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+        SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+    } else
+        rrdset_next(RS);
+
+    rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec);
+    rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec);
+    rrdset_done(RS);
+}
diff --git a/ml/ADCharts.h b/ml/ADCharts.h
new file mode 100644
index 0000000000..0be324f7d7
--- /dev/null
+++ b/ml/ADCharts.h
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef ML_ADCHARTS_H
+#define ML_ADCHARTS_H
+
+#include "ml-private.h"
+
+namespace ml {
+
+void updateDimensionsChart(RRDHOST *RH,
+                           collected_number NumTrainedDimensions,
+                           collected_number NumNormalDimensions,
+                           collected_number NumAnomalousDimensions);
+
+void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate);
+
+void updateDetectionChart(RRDHOST *RH);
+
+void updateTrainingChart(RRDHOST *RH, struct rusage *TRU);
+
+} // namespace ml
+
+#endif /* ML_ADCHARTS_H */
diff --git a/ml/BitBufferCounter.cc b/ml/BitBufferCounter.cc
deleted file mode 100644
index 5e1ab5aca3..0000000000
--- a/ml/BitBufferCounter.cc
+++ /dev/null
@@ -1,29 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "BitBufferCounter.h"
-
-using namespace ml;
-
-std::vector<bool> BitBufferCounter::getBuffer() const {
-    std::vector<bool> Buffer;
-
-    for (size_t Idx = start(); Idx != (start() + size()); Idx++)
-        Buffer.push_back(V[Idx % V.size()]);
-
-    return Buffer;
-}
-
-void BitBufferCounter::insert(bool Bit) {
-    if (N >= V.size())
-        NumSetBits -= (V[start()] == true);
-
-    NumSetBits += (Bit == true);
-    V[N++ % V.size()] = Bit;
-}
-
-void BitBufferCounter::print(std::ostream &OS) const {
-    std::vector<bool> Buffer = getBuffer();
-
-    for (bool B : Buffer)
-        OS << B;
-}
diff --git a/ml/BitBufferCounter.h b/ml/BitBufferCounter.h
deleted file mode 100644
index db924d7762..0000000000
--- a/ml/BitBufferCounter.h
+++ /dev/null
@@ -1,54 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef BIT_BUFFER_COUNTER_H
-#define BIT_BUFFER_COUNTER_H
-
-#include "ml-private.h"
-
-namespace ml {
-
-class BitBufferCounter {
-public:
-    BitBufferCounter(size_t Capacity) : V(Capacity, 0), NumSetBits(0), N(0) {}
-
-    std::vector<bool> getBuffer() const;
-
-    void insert(bool Bit);
-
-    void print(std::ostream &OS) const;
-
-    bool isFilled() const {
-       return N >= V.size();
-    }
-
-    size_t numSetBits() const {
-        return NumSetBits;
-    }
-
-private:
-    inline size_t size() const {
-        return N < V.size() ? N : V.size();
-    }
-
-    inline size_t start() const {
-        if (N <= V.size())
-            return 0;
-
-        return N % V.size();
-    }
-
-private:
-    std::vector<bool> V;
-    size_t NumSetBits;
-
-    size_t N;
-};
-
-} // namespace ml
-
-inline std::ostream& operator<<(std::ostream &OS, const ml::BitBufferCounter &BBC) {
-    BBC.print(OS);
-    return OS;
-}
-
-#endif /* BIT_BUFFER_COUNTER_H */
diff --git a/ml/BitRateWindow.cc b/ml/BitRateWindow.cc
deleted file mode 100644
index c4c994c42d..0000000000
--- a/ml/BitRateWindow.cc
+++ /dev/null
@@ -1,75 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "BitRateWindow.h"
-
-using namespace ml;
-
-std::pair<BitRateWindow::Edge, size_t> BitRateWindow::insert(bool Bit) {
-    Edge E;
-
-    BBC.insert(Bit);
-    switch (CurrState) {
-        case State::NotFilled: {
-            if (BBC.isFilled()) {
-                if (BBC.numSetBits() < SetBitsThreshold) {
-                    CurrState = State::BelowThreshold;
-                } else {
-                    CurrState = State::AboveThreshold;
-                }
-            } else {
-                CurrState = State::NotFilled;
-            }
-
-            E = {State::NotFilled, CurrState};
-            break;
-        } case State::BelowThreshold: {
-            if (BBC.numSetBits() >= SetBitsThreshold) {
-                CurrState = State::AboveThreshold;
-            }
-
-            E = {State::BelowThreshold, CurrState};
-            break;
-        } case State::AboveThreshold: {
-            if ((BBC.numSetBits() < SetBitsThreshold) ||
-                (CurrLength == MaxLength)) {
-                CurrState = State::Idle;
-            }
-
-            E = {State::AboveThreshold, CurrState};
-            break;
-        } case State::Idle: {
-            if (CurrLength == IdleLength) {
-                CurrState = State::NotFilled;
-            }
-
-            E = {State::Idle, CurrState};
-            break;
-        }
-    }
-
-    Action A = EdgeActions[E];
-    size_t L = (this->*A)(E.first, Bit);
-    return {E, L};
-}
-
-void BitRateWindow::print(std::ostream &OS) const {
-    switch (CurrState) {
-        case State::NotFilled:
-            OS << "NotFilled";
-            break;
-        case State::BelowThreshold:
-            OS << "BelowThreshold";
-            break;
-        case State::AboveThreshold:
-            OS << "AboveThreshold";
-            break;
-        case State::Idle:
-            OS << "Idle";
-            break;
-        default:
-            OS << "UnknownState";
-            break;
-    }
-
-    OS << ": " << BBC << " (Current Length: " << CurrLength << ")";
-}
diff --git a/ml/BitRateWindow.h b/ml/BitRateWindow.h
deleted file mode 100644
index 0d99008b85..0000000000
--- a/ml/BitRateWindow.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef BIT_RATE_WINDOW_H
-#define BIT_RATE_WINDOW_H
-
-#include "BitBufferCounter.h"
-#include "ml-private.h"
-
-namespace ml {
-
-class BitRateWindow {
-public:
-    enum class State {
-        NotFilled,
-        BelowThreshold,
-        AboveThreshold,
-        Idle
-    };
-
-    using Edge = std::pair<State, State>;
-    using Action = size_t (BitRateWindow::*)(State PrevState, bool NewBit);
-
-private:
-    std::map<Edge, Action> EdgeActions = {
-        // From == To
-        {
-            Edge(State::NotFilled, State::NotFilled),
-            &BitRateWindow::onRoundtripNotFilled,
-        },
-        {
-            Edge(State::BelowThreshold, State::BelowThreshold),
-            &BitRateWindow::onRoundtripBelowThreshold,
-        },
-        {
-            Edge(State::AboveThreshold, State::AboveThreshold),
-            &BitRateWindow::onRoundtripAboveThreshold,
-        },
-        {
-            Edge(State::Idle, State::Idle),
-            &BitRateWindow::onRoundtripIdle,
-        },
-
-
-        // NotFilled => {BelowThreshold, AboveThreshold}
-        {
-            Edge(State::NotFilled, State::BelowThreshold),
-            &BitRateWindow::onNotFilledToBelowThreshold
-        },
-        {
-            Edge(State::NotFilled, State::AboveThreshold),
-            &BitRateWindow::onNotFilledToAboveThreshold
-        },
-
-        // BelowThreshold => AboveThreshold
-        {
-            Edge(State::BelowThreshold, State::AboveThreshold),
-            &BitRateWindow::onBelowToAboveThreshold
-        },
-
-        // AboveThreshold => Idle
-        {
-            Edge(State::AboveThreshold, State::Idle),
-            &BitRateWindow::onAboveThresholdToIdle
-        },
-
-        // Idle => NotFilled
-        {
-            Edge(State::Idle, State::NotFilled),
-            &BitRateWindow::onIdleToNotFilled
-        },
-    };
-
-public:
-    BitRateWindow(size_t MinLength, size_t MaxLength, size_t IdleLength,
-                  size_t SetBitsThreshold) :
-        MinLength(MinLength), MaxLength(MaxLength), IdleLength(IdleLength),
-        SetBitsThreshold(SetBitsThreshold),
-        CurrState(State::NotFilled), CurrLength(0), BBC(MinLength) {}
-
-    std::pair<Edge, size_t> insert(bool Bit);
-
-    void print(std::ostream &OS) const;
-
-private:
-    size_t onRoundtripNotFilled(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength += 1;
-        return CurrLength;
-    }
-
-    size_t onRoundtripBelowThreshold(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength = MinLength;
-        return CurrLength;
-    }
-
-    size_t onRoundtripAboveThreshold(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength += 1;
-        return CurrLength;
-    }
-
-    size_t onRoundtripIdle(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength += 1;
-        return CurrLength;
-    }
-
-    size_t onNotFilledToBelowThreshold(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength = MinLength;
-        return CurrLength;
-    }
-
-    size_t onNotFilledToAboveThreshold(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength += 1;
-        return CurrLength;
-    }
-
-    size_t onBelowToAboveThreshold(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        CurrLength = MinLength;
-        return CurrLength;
-    }
-
-    size_t onAboveThresholdToIdle(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        size_t PrevLength = CurrLength;
-        CurrLength = 1;
-        return PrevLength;
-    }
-
-    size_t onIdleToNotFilled(State PrevState, bool NewBit) {
-        (void) PrevState, (void) NewBit;
-
-        BBC = BitBufferCounter(MinLength);
-        BBC.insert(NewBit);
-
-        CurrLength = 1;
-        return CurrLength;
-    }
-
-private:
-    size_t MinLength;
-    size_t MaxLength;
-    size_t IdleLength;
-    size_t SetBitsThreshold;
-
-    State CurrState;
-    size_t CurrLength;
-    BitBufferCounter BBC;
-};
-
-} // namespace ml
-
-inline std::ostream& operator<<(std::ostream &OS, const ml::BitRateWindow BRW) {
-    BRW.print(OS);
-    return OS;
-}
-
-#endif /* BIT_RATE_WINDOW_H */
diff --git a/ml/Config.cc b/ml/Config.cc
index 65b05a34d6..63b570156a 100644
--- a/ml/Config.cc
+++ b/ml/Config.cc
@@ -31,6 +31,7 @@ void Config::readMLConfig(void) {
     unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
     unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
     unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
+    unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
 
     unsigned DBEngineAnomalyRateEvery = config_get_number(ConfigSectionML, "dbengine anomaly rate every", 30);
 
@@ -42,25 +43,19 @@ void Config::readMLConfig(void) {
     unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);
 
     double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);
-    double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.01);
 
-    double ADMinWindowSize = config_get_float(ConfigSectionML, "minimum window size", 30);
-    double ADMaxWindowSize = config_get_float(ConfigSectionML, "maximum window size", 600);
-    double ADIdleWindowSize = config_get_float(ConfigSectionML, "idle window size", 30);
-    double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25);
-    double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05);
-
-    std::stringstream SS;
-    SS << netdata_configured_cache_dir << "/anomaly-detection.db";
-    Cfg.AnomalyDBPath = SS.str();
+    double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0);
+    std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection grouping method", "average");
+    time_t AnomalyDetectionQueryDuration = config_get_number(ConfigSectionML, "anomaly detection grouping duration", 5 * 60);
 
     /*
      * Clamp
      */
 
-    MaxTrainSamples = clamp(MaxTrainSamples, 1 * 3600u, 24 * 3600u);
-    MinTrainSamples = clamp(MinTrainSamples, 1 * 900u, 6 * 3600u);
-    TrainEvery = clamp(TrainEvery, 1 * 3600u, 6 * 3600u);
+    MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
+    MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
+    TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
+    NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
 
     DBEngineAnomalyRateEvery = clamp(DBEngineAnomalyRateEvery, 1 * 30u, 15 * 60u);
 
@@ -72,13 +67,9 @@ void Config::readMLConfig(void) {
     MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);
 
     DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);
-    HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.01, 1.0);
 
-    ADMinWindowSize = clamp(ADMinWindowSize, 30.0, 300.0);
-    ADMaxWindowSize = clamp(ADMaxWindowSize, 60.0, 900.0);
-    ADIdleWindowSize = clamp(ADIdleWindowSize, 30.0, 900.0);
-    ADWindowRateThreshold = clamp(ADWindowRateThreshold, 0.01, 0.99);
-    ADDimensionRateThreshold = clamp(ADDimensionRateThreshold, 0.01, 0.99);
+    HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.1, 10.0);
+    AnomalyDetectionQueryDuration = clamp<time_t>(AnomalyDetectionQueryDuration, 60, 15 * 60);
 
     /*
      * Validate
@@ -91,13 +82,6 @@ void Config::readMLConfig(void) {
         MaxTrainSamples = 4 * 3600;
     }
 
-    if (ADMinWindowSize >= ADMaxWindowSize) {
-        error("invalid min/max anomaly window size found (%lf >= %lf)", ADMinWindowSize, ADMaxWindowSize);
-
-        ADMinWindowSize = 30.0;
-        ADMaxWindowSize = 600.0;
-    }
-
     /*
      * Assign to config instance
      */
@@ -107,6 +91,7 @@ void Config::readMLConfig(void) {
     Cfg.MaxTrainSamples = MaxTrainSamples;
     Cfg.MinTrainSamples = MinTrainSamples;
     Cfg.TrainEvery = TrainEvery;
+    Cfg.NumModelsToUse = NumModelsToUse;
 
     Cfg.DBEngineAnomalyRateEvery = DBEngineAnomalyRateEvery;
 
@@ -118,13 +103,10 @@ void Config::readMLConfig(void) {
     Cfg.MaxKMeansIters = MaxKMeansIters;
 
     Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
-    Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
 
-    Cfg.ADMinWindowSize = ADMinWindowSize;
-    Cfg.ADMaxWindowSize = ADMaxWindowSize;
-    Cfg.ADIdleWindowSize = ADIdleWindowSize;
-    Cfg.ADWindowRateThreshold = ADWindowRateThreshold;
-    Cfg.ADDimensionRateThreshold = ADDimensionRateThreshold;
+    Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
+    Cfg.AnomalyDetectionGroupingMethod = web_client_api_request_v1_data_group(AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
+    Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
 
     Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
     Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
diff --git a/ml/Config.h b/ml/Config.h
index 595fd072bb..d876d4aa41 100644
--- a/ml/Config.h
+++ b/ml/Config.h
@@ -14,6 +14,7 @@ public:
     unsigned MaxTrainSamples;
     unsigned MinTrainSamples;
     unsigned TrainEvery;
+    unsigned NumModelsToUse;
 
     unsigned DBEngineAnomalyRateEvery;
 
@@ -25,13 +26,10 @@ public:
     unsigned MaxKMeansIters;
 
     double DimensionAnomalyScoreThreshold;
-    double HostAnomalyRateThreshold;
 
-    double ADMinWindowSize;
-    double ADMaxWindowSize;
-    double ADIdleWindowSize;
-    double ADWindowRateThreshold;
-    double ADDimensionRateThreshold;
+    double HostAnomalyRateThreshold;
+    RRDR_GROUPING AnomalyDetectionGroupingMethod;
+    time_t AnomalyDetectionQueryDuration;
 
     bool StreamADCharts;
 
@@ -41,7 +39,6 @@ public:
     std::string ChartsToSkip;
     SIMPLE_PATTERN *SP_ChartsToSkip;
 
-    std::string AnomalyDBPath;
     std::vector<uint32_t> RandomNums;
 
     void readMLConfig();
diff --git a/ml/Database.cc b/ml/Database.cc
deleted file mode 100644
index b46f627f6a..0000000000
--- a/ml/Database.cc
+++ /dev/null
@@ -1,127 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "Database.h"
-
-const char *ml::Database::SQL_CREATE_ANOMALIES_TABLE =
-    "CREATE TABLE IF NOT EXISTS anomaly_events( "
-    "     anomaly_detector_name text NOT NULL, "
-    "     anomaly_detector_version int NOT NULL, "