Add a chart that groups anomaly rate by chart type. (#15856)

* Add a chart that groups anomaly rate by chart type.

* Use type_anomaly_rate for context.

3 files changed, 85 insertions, 1 deletions

diff --git a/ml/ad_charts.cc b/ml/ad_charts.cc
index ca4dca1393..39ffce0123 100644
--- a/ml/ad_charts.cc
+++ b/ml/ad_charts.cc
@@ -222,7 +222,7 @@ void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats
 
 void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number AnomalyRate) {
     /*
-     * Anomaly rate
+     * Host anomaly rate
     */
     {
         if (!host->anomaly_rate_rs) {
@@ -259,6 +259,56 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
     }
 
     /*
+     * Type anomaly rate
+    */
+    {
+        if (!host->type_anomaly_rate_rs) {
+            char id_buf[1024];
+            char name_buf[1024];
+
+            snprintfz(id_buf, 1024, "type_anomaly_rate_on_%s", localhost->machine_guid);
+            snprintfz(name_buf, 1024, "type_anomaly_rate_on_%s", rrdhost_hostname(localhost));
+
+            host->type_anomaly_rate_rs = rrdset_create(
+                    host->rh,
+                    "anomaly_detection", // type
+                    id_buf, // id
+                    name_buf, // name
+                    "anomaly_rate", // family
+                    "anomaly_detection.type_anomaly_rate", // ctx
+                    "Percentage of anomalous dimensions by type", // title
+                    "percentage", // units
+                    NETDATA_ML_PLUGIN, // plugin
+                    NETDATA_ML_MODULE_DETECTION, // module
+                    ML_CHART_PRIO_TYPE_ANOMALY_RATE, // priority
+                    localhost->rrd_update_every, // update_every
+                    RRDSET_TYPE_LINE // chart_type
+            );
+
+            rrdset_flag_set(host->type_anomaly_rate_rs, RRDSET_FLAG_ANOMALY_DETECTION);
+        }
+
+        for (auto &entry : host->type_anomaly_rate) {
+            ml_type_anomaly_rate_t &type_anomaly_rate = entry.second;
+
+            if (!type_anomaly_rate.rd)
+                type_anomaly_rate.rd = rrddim_add(host->type_anomaly_rate_rs, string2str(entry.first), NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
+
+            double ar = 0.0;
+            size_t n = type_anomaly_rate.anomalous_dimensions + type_anomaly_rate.normal_dimensions;
+            if (n)
+                ar = static_cast<double>(type_anomaly_rate.anomalous_dimensions) / n;
+
+            rrddim_set_by_pointer(host->type_anomaly_rate_rs, type_anomaly_rate.rd, ar * 10000.0);
+
+            type_anomaly_rate.anomalous_dimensions = 0;
+            type_anomaly_rate.normal_dimensions = 0;
+        }
+
+        rrdset_done(host->type_anomaly_rate_rs);
+    }
+
+    /*
      * Detector Events
     */
     {
diff --git a/ml/ml-private.h b/ml/ml-private.h
index 5216d57cd0..f373456fa0 100644
--- a/ml/ml-private.h
+++ b/ml/ml-private.h
@@ -8,6 +8,7 @@
 
 #include <vector>
 #include <queue>
+#include <unordered_map>
 
 typedef double calculated_number_t;
 typedef dlib::matrix<calculated_number_t, 6, 1> DSample;
@@ -211,6 +212,12 @@ typedef struct {
 void ml_chart_update_dimension(ml_chart_t *chart, ml_dimension_t *dim, bool is_anomalous);
 
 typedef struct {
+    RRDDIM *rd;
+    size_t normal_dimensions;
+    size_t anomalous_dimensions;
+} ml_type_anomaly_rate_t;
+
+typedef struct {
     RRDHOST *rh;
 
     std::atomic<bool> ml_running;
@@ -255,6 +262,9 @@ typedef struct {
     RRDSET *detector_events_rs;
     RRDDIM *detector_events_above_threshold_rd;
     RRDDIM *detector_events_new_anomaly_event_rd;
+
+    RRDSET *type_anomaly_rate_rs;
+    std::unordered_map<STRING *, ml_type_anomaly_rate_t> type_anomaly_rate;
 } ml_host_t;
 
 typedef struct {
diff --git a/ml/ml.cc b/ml/ml.cc
index 58956b2c6f..867ffee7b4 100644
--- a/ml/ml.cc
+++ b/ml/ml.cc
@@ -1082,6 +1082,21 @@ ml_host_detect_once(ml_host_t *host)
 
             host->mls.num_anomalous_dimensions += chart_mls.num_anomalous_dimensions;
             host->mls.num_normal_dimensions += chart_mls.num_normal_dimensions;
+
+            STRING *key = rs->parts.type;
+            auto &um = host->type_anomaly_rate;
+            auto it = um.find(key);
+            if (it == um.end()) {
+                um[key] = ml_type_anomaly_rate_t {
+                    .rd = NULL,
+                    .normal_dimensions = 0,
+                    .anomalous_dimensions = 0
+                };
+                it = um.find(key);
+            }
+
+            it->second.anomalous_dimensions += chart_mls.num_anomalous_dimensions;
+            it->second.normal_dimensions += chart_mls.num_normal_dimensions;
         }
         rrdset_foreach_done(rsp);
 
@@ -1095,6 +1110,15 @@ ml_host_detect_once(ml_host_t *host)
         netdata_mutex_unlock(&host->mutex);
     } else {
         host->host_anomaly_rate = 0.0;
+
+        auto &um = host->type_anomaly_rate;
+        for (auto &entry: um) {
+            entry.second = ml_type_anomaly_rate_t {
+                .rd = NULL,
+                .normal_dimensions = 0,
+                .anomalous_dimensions = 0
+            };
+        }
     }
 
     worker_is_busy(WORKER_JOB_DETECTION_DIM_CHART);