diff options
-rw-r--r-- | .gitmodules | 2 | ||||
-rw-r--r-- | CMakeLists.txt | 20 | ||||
-rw-r--r-- | Makefile.am | 26 | ||||
-rw-r--r-- | configure.ac | 6 | ||||
-rw-r--r-- | ml/ADCharts.cc | 236 | ||||
-rw-r--r-- | ml/ADCharts.h | 23 | ||||
-rw-r--r-- | ml/BitBufferCounter.cc | 29 | ||||
-rw-r--r-- | ml/BitBufferCounter.h | 54 | ||||
-rw-r--r-- | ml/BitRateWindow.cc | 75 | ||||
-rw-r--r-- | ml/BitRateWindow.h | 170 | ||||
-rw-r--r-- | ml/Config.cc | 46 | ||||
-rw-r--r-- | ml/Config.h | 11 | ||||
-rw-r--r-- | ml/Database.cc | 127 | ||||
-rw-r--r-- | ml/Database.h | 131 | ||||
-rw-r--r-- | ml/Dimension.cc | 86 | ||||
-rw-r--r-- | ml/Dimension.h | 169 | ||||
-rw-r--r-- | ml/Host.cc | 352 | ||||
-rw-r--r-- | ml/Host.h | 30 | ||||
-rw-r--r-- | ml/KMeans.cc | 43 | ||||
-rw-r--r-- | ml/KMeans.h (renamed from ml/kmeans/KMeans.h) | 15 | ||||
-rw-r--r-- | ml/Makefile.am | 8 | ||||
-rw-r--r-- | ml/Query.h | 13 | ||||
-rw-r--r-- | ml/SamplesBuffer.cc (renamed from ml/kmeans/SamplesBuffer.cc) | 0 | ||||
-rw-r--r-- | ml/SamplesBuffer.h (renamed from ml/kmeans/SamplesBuffer.h) | 0 | ||||
-rw-r--r-- | ml/SamplesBufferTests.cc (renamed from ml/kmeans/Tests.cc) | 9 | ||||
-rw-r--r-- | ml/Tests.cc | 301 | ||||
m--------- | ml/dlib (renamed from ml/kmeans/dlib) | 0 | ||||
-rw-r--r-- | ml/kmeans/KMeans.cc | 55 | ||||
-rw-r--r-- | ml/kmeans/Makefile.am | 4 | ||||
-rw-r--r-- | ml/ml-dummy.c | 24 | ||||
-rw-r--r-- | ml/ml-private.h | 2 | ||||
-rw-r--r-- | ml/ml.cc | 61 | ||||
-rw-r--r-- | ml/ml.h | 7 | ||||
-rw-r--r-- | web/api/web_api_v1.c | 88 |
34 files changed, 565 insertions, 1658 deletions
diff --git a/.gitmodules b/.gitmodules index 0f01d3d680..d3c7ace405 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ path = aclk/aclk-schemas url = https://github.com/netdata/aclk-schemas.git [submodule "ml/kmeans/dlib"] - path = ml/kmeans/dlib + path = ml/dlib url = https://github.com/davisking/dlib.git shallow = true ignore = dirty diff --git a/CMakeLists.txt b/CMakeLists.txt index fc7e47fbd0..e0c114ab4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -348,11 +348,11 @@ ENDIF() # Detect ml dependencies file(STRINGS "${CMAKE_SOURCE_DIR}/config.h" DEFINE_ENABLE_ML REGEX "^#define ENABLE_ML 1$") IF(DEFINE_ENABLE_ML MATCHES ".+" AND - EXISTS "${CMAKE_SOURCE_DIR}/ml/kmeans/dlib/dlib/all/source.cpp" AND + EXISTS "${CMAKE_SOURCE_DIR}/ml/dlib/dlib/all/source.cpp" AND EXISTS "${CMAKE_SOURCE_DIR}/ml/json/single_include/nlohmann/json.hpp") set(ENABLE_ML True) list(APPEND NETDATA_COMMON_CFLAGS "-DDLIB_NO_GUI_SUPPORT") - list(APPEND NETDATA_COMMON_INCLUDE_DIRS "ml/kmeans/dlib") + list(APPEND NETDATA_COMMON_INCLUDE_DIRS "ml/dlib") ELSE() set(ENABLE_ML False) ENDIF() @@ -1025,24 +1025,18 @@ set(ML_FILES IF(ENABLE_ML) list(APPEND ML_FILES - ml/BitBufferCounter.h - ml/BitBufferCounter.cc - ml/BitRateWindow.h - ml/BitRateWindow.cc ml/Config.h ml/Config.cc - ml/Database.h - ml/Database.cc ml/Dimension.cc ml/Dimension.h ml/Host.h ml/Host.cc ml/Query.h - ml/kmeans/KMeans.h - ml/kmeans/KMeans.cc - ml/kmeans/SamplesBuffer.h - ml/kmeans/SamplesBuffer.cc - ml/kmeans/dlib/dlib/all/source.cpp + ml/KMeans.h + ml/KMeans.cc + ml/SamplesBuffer.h + ml/SamplesBuffer.cc + ml/dlib/dlib/all/source.cpp ml/json/single_include/nlohmann/json.hpp ml/ml.cc ml/ml-private.h diff --git a/Makefile.am b/Makefile.am index c5e3e0949d..34026e9888 100644 --- a/Makefile.am +++ b/Makefile.am @@ -38,7 +38,7 @@ EXTRA_DIST = \ build/m4/ax_c_mallopt.m4 \ build/m4/tcmalloc.m4 \ build/m4/ax_c__generic.m4 \ - ml/kmeans/dlib \ + ml/dlib \ README.md \ LICENSE \ REDISTRIBUTED.md \ @@ -109,7 +109,6 @@ SUBDIRS += \ claim \ parser \ spawn \ - ml \ $(NULL) AM_CFLAGS = \ @@ -237,39 +236,34 @@ ML_FILES = \ if ENABLE_ML ML_FILES += \ - ml/BitBufferCounter.h \ - ml/BitBufferCounter.cc \ - ml/BitRateWindow.h \ - ml/BitRateWindow.cc \ + ml/ADCharts.h \ + ml/ADCharts.cc \ ml/Config.h \ ml/Config.cc \ - ml/Database.h \ - ml/Database.cc \ ml/Dimension.cc \ ml/Dimension.h \ ml/Host.h \ ml/Host.cc \ ml/Query.h \ - ml/kmeans/KMeans.h \ - ml/kmeans/KMeans.cc \ - ml/kmeans/SamplesBuffer.h \ - ml/kmeans/SamplesBuffer.cc \ - ml/kmeans/dlib/dlib/all/source.cpp \ + ml/KMeans.h \ + ml/KMeans.cc \ + ml/SamplesBuffer.h \ + ml/SamplesBuffer.cc \ + ml/dlib/dlib/all/source.cpp \ ml/json/single_include/nlohmann/json.hpp \ ml/ml.cc \ ml/ml-private.h \ $(NULL) # Disable warnings from dlib library -ml/kmeans/dlib/dlib/all/source.$(OBJEXT) : CXXFLAGS += -Wno-sign-compare -Wno-type-limits -Wno-aggressive-loop-optimizations -Wno-stringop-overflow +ml/dlib/dlib/all/source.$(OBJEXT) : CXXFLAGS += -Wno-sign-compare -Wno-type-limits -Wno-aggressive-loop-optimizations -Wno-stringop-overflow endif if ENABLE_ML_TESTS ML_TESTS_FILES = \ - ml/kmeans/Tests.cc \ - ml/Tests.cc \ + ml/SamplesBufferTests.cc \ $(NULL) endif diff --git a/configure.ac b/configure.ac index d9cf075138..ef060a91f8 100644 --- a/configure.ac +++ b/configure.ac @@ -1109,7 +1109,7 @@ fi # Check if submodules have not been fetched. Fail if ML was explicitly requested. AC_MSG_CHECKING([if git submodules are present for machine learning functionality]) -if test -f "ml/kmeans/dlib/dlib/all/source.cpp" -a -f "ml/json/single_include/nlohmann/json.hpp"; then +if test -f "ml/dlib/dlib/all/source.cpp" -a -f "ml/json/single_include/nlohmann/json.hpp"; then AC_MSG_RESULT([yes]) have_ml_submodules="yes" else @@ -1149,7 +1149,7 @@ fi AM_CONDITIONAL([ENABLE_ML], [test "${build_ml}" = "yes"]) if test "${build_ml}" = "yes"; then AC_DEFINE([ENABLE_ML], [1], [anomaly detection usability]) - OPTIONAL_ML_CFLAGS="-DDLIB_NO_GUI_SUPPORT -I \$(abs_top_srcdir)/ml/kmeans/dlib" + OPTIONAL_ML_CFLAGS="-DDLIB_NO_GUI_SUPPORT -I \$(abs_top_srcdir)/ml/dlib" OPTIONAL_ML_LIBS="" fi @@ -1695,8 +1695,6 @@ AC_CONFIG_FILES([ exporting/tests/Makefile health/Makefile health/notifications/Makefile - ml/Makefile - ml/kmeans/Makefile libnetdata/Makefile libnetdata/tests/Makefile libnetdata/adaptive_resortable_list/Makefile diff --git a/ml/ADCharts.cc b/ml/ADCharts.cc new file mode 100644 index 0000000000..5a78b97918 --- /dev/null +++ b/ml/ADCharts.cc @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ADCharts.h" +#include "Config.h" + +void ml::updateDimensionsChart(RRDHOST *RH, + collected_number NumTrainedDimensions, + collected_number NumNormalDimensions, + collected_number NumAnomalousDimensions) { + static thread_local RRDSET *RS = nullptr; + static thread_local RRDDIM *NumTotalDimensionsRD = nullptr; + static thread_local RRDDIM *NumTrainedDimensionsRD = nullptr; + static thread_local RRDDIM *NumNormalDimensionsRD = nullptr; + static thread_local RRDDIM *NumAnomalousDimensionsRD = nullptr; + + if (!RS) { + std::stringstream IdSS, NameSS; + + IdSS << "dimensions_on_" << localhost->machine_guid; + NameSS << "dimensions_on_" << localhost->hostname; + + RS = rrdset_create( + RH, + "anomaly_detection", // type + IdSS.str().c_str(), // id + NameSS.str().c_str(), // name + "dimensions", // family + "anomaly_detection.dimensions", // ctx + "Anomaly detection dimensions", // title + "dimensions", // units + "netdata", // plugin + "ml", // module + 39183, // priority + RH->rrd_update_every, // update_every + RRDSET_TYPE_LINE // chart_type + ); + rrdset_flag_set(RS, RRDSET_FLAG_ANOMALY_DETECTION); + + NumTotalDimensionsRD = rrddim_add(RS, "total", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + NumTrainedDimensionsRD = rrddim_add(RS, "trained", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + NumNormalDimensionsRD = rrddim_add(RS, "normal", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + NumAnomalousDimensionsRD = rrddim_add(RS, "anomalous", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(RS); + + rrddim_set_by_pointer(RS, NumTotalDimensionsRD, NumNormalDimensions + NumAnomalousDimensions); + rrddim_set_by_pointer(RS, NumTrainedDimensionsRD, NumTrainedDimensions); + rrddim_set_by_pointer(RS, NumNormalDimensionsRD, NumNormalDimensions); + rrddim_set_by_pointer(RS, NumAnomalousDimensionsRD, NumAnomalousDimensions); + + rrdset_done(RS); +} + +void ml::updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate) { + static thread_local RRDSET *HostRateRS = nullptr; + static thread_local RRDDIM *AnomalyRateRD = nullptr; + + if (!HostRateRS) { + std::stringstream IdSS, NameSS; + + IdSS << "anomaly_rate_on_" << localhost->machine_guid; + NameSS << "anomaly_rate_on_" << localhost->hostname; + + HostRateRS = rrdset_create( + RH, + "anomaly_detection", // type + IdSS.str().c_str(), // id + NameSS.str().c_str(), // name + "anomaly_rate", // family + "anomaly_detection.anomaly_rate", // ctx + "Percentage of anomalous dimensions", // title + "percentage", // units + "netdata", // plugin + "ml", // module + 39184, // priority + RH->rrd_update_every, // update_every + RRDSET_TYPE_LINE // chart_type + ); + rrdset_flag_set(HostRateRS, RRDSET_FLAG_ANOMALY_DETECTION); + + AnomalyRateRD = rrddim_add(HostRateRS, "anomaly_rate", NULL, + 1, 100, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(HostRateRS); + + rrddim_set_by_pointer(HostRateRS, AnomalyRateRD, AnomalyRate); + rrdset_done(HostRateRS); + + static thread_local RRDSET *AnomalyDetectionRS = nullptr; + static thread_local RRDDIM *AboveThresholdRD = nullptr; + static thread_local RRDDIM *NewAnomalyEventRD = nullptr; + + if (!AnomalyDetectionRS) { + std::stringstream IdSS, NameSS; + + IdSS << "anomaly_detection_on_" << localhost->machine_guid; + NameSS << "anomaly_detection_on_" << localhost->hostname; + + AnomalyDetectionRS = rrdset_create( + RH, + "anomaly_detection", // type + IdSS.str().c_str(), // id + NameSS.str().c_str(), // name + "anomaly_detection", // family + "anomaly_detection.detector_events", // ctx + "Anomaly detection events", // title + "percentage", // units + "netdata", // plugin + "ml", // module + 39185, // priority + RH->rrd_update_every, // update_every + RRDSET_TYPE_LINE // chart_type + ); + rrdset_flag_set(AnomalyDetectionRS, RRDSET_FLAG_ANOMALY_DETECTION); + + AboveThresholdRD = rrddim_add(AnomalyDetectionRS, "above_threshold", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + NewAnomalyEventRD = rrddim_add(AnomalyDetectionRS, "new_anomaly_event", NULL, + 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(AnomalyDetectionRS); + + /* + * Compute the values of the dimensions based on the host rate chart + */ + ONEWAYALLOC *OWA = onewayalloc_create(0); + time_t Now = now_realtime_sec(); + time_t Before = Now - RH->rrd_update_every; + time_t After = Before - Cfg.AnomalyDetectionQueryDuration; + RRDR_OPTIONS Options = static_cast<RRDR_OPTIONS>(0x00000000); + + RRDR *R = rrd2rrdr( + OWA, HostRateRS, + 1 /* points wanted */, + After, + Before, + Cfg.AnomalyDetectionGroupingMethod, + 0 /* resampling time */, + Options, "anomaly_rate", + NULL /* context param list */, + NULL /* group options */, + 0, /* timeout */ + 0 /* tier */ + ); + assert(R->d == 1 && R->n == 1 && R->rows == 1); + + static thread_local bool PrevAboveThreshold = false; + bool AboveThreshold = R->v[0] >= Cfg.HostAnomalyRateThreshold; + bool NewAnomalyEvent = AboveThreshold && !PrevAboveThreshold; + PrevAboveThreshold = AboveThreshold; + + rrddim_set_by_pointer(AnomalyDetectionRS, AboveThresholdRD, AboveThreshold); + rrddim_set_by_pointer(AnomalyDetectionRS, NewAnomalyEventRD, NewAnomalyEvent); + rrdset_done(AnomalyDetectionRS); + + rrdr_free(OWA, R); + onewayalloc_destroy(OWA); +} + +void ml::updateDetectionChart(RRDHOST *RH) { + static thread_local RRDSET *RS = nullptr; + static thread_local RRDDIM *UserRD, *SystemRD = nullptr; + + if (!RS) { + std::stringstream IdSS, NameSS; + + IdSS << "prediction_stats_" << RH->machine_guid; + NameSS << "prediction_stats_for_" << RH->hostname; + + RS = rrdset_create_localhost( + "netdata", // type + IdSS.str().c_str(), // id + NameSS.str().c_str(), // name + "ml", // family + "netdata.prediction_stats", // ctx + "Prediction thread CPU usage", // title + "milliseconds/s", // units + "netdata", // plugin + "ml", // module + 136000, // priority + RH->rrd_update_every, // update_every + RRDSET_TYPE_STACKED // chart_type + ); + + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(RS); + + struct rusage TRU; + getrusage(RUSAGE_THREAD, &TRU); + + rrddim_set_by_pointer(RS, UserRD, TRU.ru_utime.tv_sec * 1000000ULL + TRU.ru_utime.tv_usec); + rrddim_set_by_pointer(RS, SystemRD, TRU.ru_stime.tv_sec * 1000000ULL + TRU.ru_stime.tv_usec); + rrdset_done(RS); +} + +void ml::updateTrainingChart(RRDHOST *RH, struct rusage *TRU) { + static thread_local RRDSET *RS = nullptr; + static thread_local RRDDIM *UserRD = nullptr; + static thread_local RRDDIM *SystemRD = nullptr; + + if (!RS) { + std::stringstream IdSS, NameSS; + + IdSS << "training_stats_" << RH->machine_guid; + NameSS << "training_stats_for_" << RH->hostname; + + RS = rrdset_create_localhost( + "netdata", // type + IdSS.str().c_str(), // id + NameSS.str().c_str(), // name + "ml", // family + "netdata.training_stats", // ctx + "Training thread CPU usage", // title + "milliseconds/s", // units + "netdata", // plugin + "ml", // module + 136001, // priority + RH->rrd_update_every, // update_every + RRDSET_TYPE_STACKED // chart_type + ); + + UserRD = rrddim_add(RS, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + SystemRD = rrddim_add(RS, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(RS); + + rrddim_set_by_pointer(RS, UserRD, TRU->ru_utime.tv_sec * 1000000ULL + TRU->ru_utime.tv_usec); + rrddim_set_by_pointer(RS, SystemRD, TRU->ru_stime.tv_sec * 1000000ULL + TRU->ru_stime.tv_usec); + rrdset_done(RS); +} diff --git a/ml/ADCharts.h b/ml/ADCharts.h new file mode 100644 index 0000000000..0be324f7d7 --- /dev/null +++ b/ml/ADCharts.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef ML_ADCHARTS_H +#define ML_ADCHARTS_H + +#include "ml-private.h" + +namespace ml { + +void updateDimensionsChart(RRDHOST *RH, + collected_number NumTrainedDimensions, + collected_number NumNormalDimensions, + collected_number NumAnomalousDimensions); + +void updateHostAndDetectionRateCharts(RRDHOST *RH, collected_number AnomalyRate); + +void updateDetectionChart(RRDHOST *RH); + +void updateTrainingChart(RRDHOST *RH, struct rusage *TRU); + +} // namespace ml + +#endif /* ML_ADCHARTS_H */ diff --git a/ml/BitBufferCounter.cc b/ml/BitBufferCounter.cc deleted file mode 100644 index 5e1ab5aca3..0000000000 --- a/ml/BitBufferCounter.cc +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "BitBufferCounter.h" - -using namespace ml; - -std::vector<bool> BitBufferCounter::getBuffer() const { - std::vector<bool> Buffer; - - for (size_t Idx = start(); Idx != (start() + size()); Idx++) - Buffer.push_back(V[Idx % V.size()]); - - return Buffer; -} - -void BitBufferCounter::insert(bool Bit) { - if (N >= V.size()) - NumSetBits -= (V[start()] == true); - - NumSetBits += (Bit == true); - V[N++ % V.size()] = Bit; -} - -void BitBufferCounter::print(std::ostream &OS) const { - std::vector<bool> Buffer = getBuffer(); - - for (bool B : Buffer) - OS << B; -} diff --git a/ml/BitBufferCounter.h b/ml/BitBufferCounter.h deleted file mode 100644 index db924d7762..0000000000 --- a/ml/BitBufferCounter.h +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef BIT_BUFFER_COUNTER_H -#define BIT_BUFFER_COUNTER_H - -#include "ml-private.h" - -namespace ml { - -class BitBufferCounter { -public: - BitBufferCounter(size_t Capacity) : V(Capacity, 0), NumSetBits(0), N(0) {} - - std::vector<bool> getBuffer() const; - - void insert(bool Bit); - - void print(std::ostream &OS) const; - - bool isFilled() const { - return N >= V.size(); - } - - size_t numSetBits() const { - return NumSetBits; - } - -private: - inline size_t size() const { - return N < V.size() ? N : V.size(); - } - - inline size_t start() const { - if (N <= V.size()) - return 0; - - return N % V.size(); - } - -private: - std::vector<bool> V; - size_t NumSetBits; - - size_t N; -}; - -} // namespace ml - -inline std::ostream& operator<<(std::ostream &OS, const ml::BitBufferCounter &BBC) { - BBC.print(OS); - return OS; -} - -#endif /* BIT_BUFFER_COUNTER_H */ diff --git a/ml/BitRateWindow.cc b/ml/BitRateWindow.cc deleted file mode 100644 index c4c994c42d..0000000000 --- a/ml/BitRateWindow.cc +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "BitRateWindow.h" - -using namespace ml; - -std::pair<BitRateWindow::Edge, size_t> BitRateWindow::insert(bool Bit) { - Edge E; - - BBC.insert(Bit); - switch (CurrState) { - case State::NotFilled: { - if (BBC.isFilled()) { - if (BBC.numSetBits() < SetBitsThreshold) { - CurrState = State::BelowThreshold; - } else { - CurrState = State::AboveThreshold; - } - } else { - CurrState = State::NotFilled; - } - - E = {State::NotFilled, CurrState}; - break; - } case State::BelowThreshold: { - if (BBC.numSetBits() >= SetBitsThreshold) { - CurrState = State::AboveThreshold; - } - - E = {State::BelowThreshold, CurrState}; - break; - } case State::AboveThreshold: { - if ((BBC.numSetBits() < SetBitsThreshold) || - (CurrLength == MaxLength)) { - CurrState = State::Idle; - } - - E = {State::AboveThreshold, CurrState}; - break; - } case State::Idle: { - if (CurrLength == IdleLength) { - CurrState = State::NotFilled; - } - - E = {State::Idle, CurrState}; - break; - } - } - - Action A = EdgeActions[E]; - size_t L = (this->*A)(E.first, Bit); - return {E, L}; -} - -void BitRateWindow::print(std::ostream &OS) const { - switch (CurrState) { - case State::NotFilled: - OS << "NotFilled"; - break; - case State::BelowThreshold: - OS << "BelowThreshold"; - break; - case State::AboveThreshold: - OS << "AboveThreshold"; - break; - case State::Idle: - OS << "Idle"; - break; - default: - OS << "UnknownState"; - break; - } - - OS << ": " << BBC << " (Current Length: " << CurrLength << ")"; -} diff --git a/ml/BitRateWindow.h b/ml/BitRateWindow.h deleted file mode 100644 index 0d99008b85..0000000000 --- a/ml/BitRateWindow.h +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef BIT_RATE_WINDOW_H -#define BIT_RATE_WINDOW_H - -#include "BitBufferCounter.h" -#include "ml-private.h" - -namespace ml { - -class BitRateWindow { -public: - enum class State { - NotFilled, - BelowThreshold, - AboveThreshold, - Idle - }; - - using Edge = std::pair<State, State>; - using Action = size_t (BitRateWindow::*)(State PrevState, bool NewBit); - -private: - std::map<Edge, Action> EdgeActions = { - // From == To - { - Edge(State::NotFilled, State::NotFilled), - &BitRateWindow::onRoundtripNotFilled, - }, - { - Edge(State::BelowThreshold, State::BelowThreshold), - &BitRateWindow::onRoundtripBelowThreshold, - }, - { - Edge(State::AboveThreshold, State::AboveThreshold), - &BitRateWindow::onRoundtripAboveThreshold, - }, - { - Edge(State::Idle, State::Idle), - &BitRateWindow::onRoundtripIdle, - }, - - - // NotFilled => {BelowThreshold, AboveThreshold} - { - Edge(State::NotFilled, State::BelowThreshold), - &BitRateWindow::onNotFilledToBelowThreshold - }, - { - Edge(State::NotFilled, State::AboveThreshold), - &BitRateWindow::onNotFilledToAboveThreshold - }, - - // BelowThreshold => AboveThreshold - { - Edge(State::BelowThreshold, State::AboveThreshold), - &BitRateWindow::onBelowToAboveThreshold - }, - - // AboveThreshold => Idle - { - Edge(State::AboveThreshold, State::Idle), - &BitRateWindow::onAboveThresholdToIdle - }, - - // Idle => NotFilled - { - Edge(State::Idle, State::NotFilled), - &BitRateWindow::onIdleToNotFilled - }, - }; - -public: - BitRateWindow(size_t MinLength, size_t MaxLength, size_t IdleLength, - size_t SetBitsThreshold) : - MinLength(MinLength), MaxLength(MaxLength), IdleLength(IdleLength), - SetBitsThreshold(SetBitsThreshold), - CurrState(State::NotFilled), CurrLength(0), BBC(MinLength) {} - - std::pair<Edge, size_t> insert(bool Bit); - - void print(std::ostream &OS) const; - -private: - size_t onRoundtripNotFilled(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength += 1; - return CurrLength; - } - - size_t onRoundtripBelowThreshold(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength = MinLength; - return CurrLength; - } - - size_t onRoundtripAboveThreshold(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength += 1; - return CurrLength; - } - - size_t onRoundtripIdle(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength += 1; - return CurrLength; - } - - size_t onNotFilledToBelowThreshold(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength = MinLength; - return CurrLength; - } - - size_t onNotFilledToAboveThreshold(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength += 1; - return CurrLength; - } - - size_t onBelowToAboveThreshold(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - CurrLength = MinLength; - return CurrLength; - } - - size_t onAboveThresholdToIdle(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - size_t PrevLength = CurrLength; - CurrLength = 1; - return PrevLength; - } - - size_t onIdleToNotFilled(State PrevState, bool NewBit) { - (void) PrevState, (void) NewBit; - - BBC = BitBufferCounter(MinLength); - BBC.insert(NewBit); - - CurrLength = 1; - return CurrLength; - } - -private: - size_t MinLength; - size_t MaxLength; - size_t IdleLength; - size_t SetBitsThreshold; - - State CurrState; - size_t CurrLength; - BitBufferCounter BBC; -}; - -} // namespace ml - -inline std::ostream& operator<<(std::ostream &OS, const ml::BitRateWindow BRW) { - BRW.print(OS); - return OS; -} - -#endif /* BIT_RATE_WINDOW_H */ diff --git a/ml/Config.cc b/ml/Config.cc index 65b05a34d6..63b570156a 100644 --- a/ml/Config.cc +++ b/ml/Config.cc @@ -31,6 +31,7 @@ void Config::readMLConfig(void) { unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600); unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900); unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600); + unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24); unsigned DBEngineAnomalyRateEvery = config_get_number(ConfigSectionML, "dbengine anomaly rate every", 30); @@ -42,25 +43,19 @@ void Config::readMLConfig(void) { unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000); double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99); - double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.01); - double ADMinWindowSize = config_get_float(ConfigSectionML, "minimum window size", 30); - double ADMaxWindowSize = config_get_float(ConfigSectionML, "maximum window size", 600); - double ADIdleWindowSize = config_get_float(ConfigSectionML, "idle window size", 30); - double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25); - double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05); - - std::stringstream SS; - SS << netdata_configured_cache_dir << "/anomaly-detection.db"; - Cfg.AnomalyDBPath = SS.str(); + double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0); + std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection |