diff options
author | vkalintiris <vasilis@netdata.cloud> | 2022-10-05 10:11:12 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-05 10:11:12 +0300 |
commit | 6850878e697d66dc90b9af1e750b22238c63c292 (patch) | |
tree | 1e4bf33a393c258ac31bf6971f1ea2b246e1635c /ml/SamplesBuffer.h | |
parent | 2b7a964d49df6deda32bffbe6141ec53429d68fd (diff) |
Remove anomaly detector (#13657)
* Move all dims under one class.
* Dimension owns anomaly rate RD.
* Remove Dimension::isAnomalous()
* Remove Dimension::trainEvery()
* Rm ml/kmeans
* Remove anomaly detector
The same logic can be implemented by using the host anomaly rate dim.
* Profile plugin.
* Revert "Profile plugin."
This reverts commit e3db37cb49c514502c5216cfe7bca2a003fb90f1.
* Add separate source files for anomaly detection charts.
* Handle training/prediction sync at the dimension level.
* Keep multiple KMeans models in mem.
* Move feature extraction outside KMeans class.
* Use multiple models.
* Add /api/v1/ml_models endpoint.
* Remove Dimension::getID()
* Use just 1 model and fix tests.
* Add detection logic based on rrdr.
* Remove config options related to anomaly detection.
* Make anomaly detection queries configurable.
* Fix ad query duration option.
* Finalize queries in all code paths.
* Check if query was initialized before finalizing it
* Do not leak OWA
* Profile plugin.
* Revert "Profile plugin."
This reverts commit 5c77145d0df7e091d030476c480ab8d9cbceb89e.
* Change context from anomaly_detection to detector_events.
Diffstat (limited to 'ml/SamplesBuffer.h')
-rw-r--r-- | ml/SamplesBuffer.h | 146 |
1 files changed, 146 insertions, 0 deletions
diff --git a/ml/SamplesBuffer.h b/ml/SamplesBuffer.h new file mode 100644 index 0000000000..1c7215ccad --- /dev/null +++ b/ml/SamplesBuffer.h @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef SAMPLES_BUFFER_H +#define SAMPLES_BUFFER_H + +#include <iostream> +#include <vector> + +#include <cassert> +#include <cstdlib> +#include <cstring> + +#include <dlib/matrix.h> + +typedef double CalculatedNumber; +typedef dlib::matrix<CalculatedNumber, 0, 1> DSample; + +class Sample { +public: + Sample(CalculatedNumber *Buf, size_t N) : CNs(Buf), NumDims(N) {} + + void initDSample(DSample &DS) const { + for (size_t Idx = 0; Idx != NumDims; Idx++) { + DS(Idx) = std::abs(CNs[Idx]); + } + } + + void add(const Sample &RHS) const { + assert(NumDims == RHS.NumDims); + + for (size_t Idx = 0; Idx != NumDims; Idx++) + CNs[Idx] += RHS.CNs[Idx]; + }; + + void diff(const Sample &RHS) const { + assert(NumDims == RHS.NumDims); + + for (size_t Idx = 0; Idx != NumDims; Idx++) + CNs[Idx] -= RHS.CNs[Idx]; + }; + + void copy(const Sample &RHS) const { + assert(NumDims == RHS.NumDims); + + std::memcpy(CNs, RHS.CNs, NumDims * sizeof(CalculatedNumber)); + } + + void scale(CalculatedNumber Factor) { + for (size_t Idx = 0; Idx != NumDims; Idx++) + CNs[Idx] *= Factor; + } + + void lag(const Sample &S, size_t LagN) { + size_t N = S.NumDims; + + for (size_t Idx = 0; Idx != (LagN + 1); Idx++) { + Sample Src(S.CNs - (Idx * N), N); + Sample Dst(CNs + (Idx * N), N); + Dst.copy(Src); + } + } + + const CalculatedNumber *getCalculatedNumbers() const { + return CNs; + }; + + void print(std::ostream &OS) const; + +private: + CalculatedNumber *CNs; + size_t NumDims; +}; + +inline std::ostream& operator<<(std::ostream &OS, const Sample &S) { + S.print(OS); + return OS; +} + +class SamplesBuffer { +public: + SamplesBuffer(CalculatedNumber *CNs, + size_t NumSamples, size_t NumDimsPerSample, + size_t DiffN, size_t SmoothN, size_t LagN, + double SamplingRatio, std::vector<uint32_t> &RandNums) : + CNs(CNs), NumSamples(NumSamples), NumDimsPerSample(NumDimsPerSample), + DiffN(DiffN), SmoothN(SmoothN), LagN(LagN), + SamplingRatio(SamplingRatio), RandNums(RandNums), + BytesPerSample(NumDimsPerSample * sizeof(CalculatedNumber)), + Preprocessed(false) {}; + + std::vector<DSample> preprocess(); + std::vector<Sample> getPreprocessedSamples() const; + + size_t capacity() const { return NumSamples; } + void print(std::ostream &OS) const; + +private: + size_t getSampleOffset(size_t Index) const { + assert(Index < NumSamples); + return Index * NumDimsPerSample; + } + + size_t getPreprocessedSampleOffset(size_t Index) const { + assert(Index < NumSamples); + return getSampleOffset(Index) * (LagN + 1); + } + + void setSample(size_t Index, const Sample &S) const { + size_t Offset = getSampleOffset(Index); + std::memcpy(&CNs[Offset], S.getCalculatedNumbers(), BytesPerSample); + } + + const Sample getSample(size_t Index) const { + size_t Offset = getSampleOffset(Index); + return Sample(&CNs[Offset], NumDimsPerSample); + }; + + const Sample getPreprocessedSample(size_t Index) const { + size_t Offset = getPreprocessedSampleOffset(Index); + return Sample(&CNs[Offset], NumDimsPerSample * (LagN + 1)); + }; + + void diffSamples(); + void smoothSamples(); + void lagSamples(); + +private: + CalculatedNumber *CNs; + size_t NumSamples; + size_t NumDimsPerSample; + size_t DiffN; + size_t SmoothN; + size_t LagN; + double SamplingRatio; + std::vector<uint32_t> &RandNums; + + size_t BytesPerSample; + bool Preprocessed; +}; + +inline std::ostream& operator<<(std::ostream& OS, const SamplesBuffer &SB) { + SB.print(OS); + return OS; +} + +#endif /* SAMPLES_BUFFER_H */ |