Refactor ML code and add support for multiple KMeans models. (#14065)

* Add profile.plugin

Creates the specified number of charts/dimensions, and supports
backfilling with pseudo-historical data.

* Bump

* Remove wrongly merged line.

* Use the number of models specified from the config section.

* Add option to consult all ML models.

* Remove profiling option consuming all models.

* Add underscore after chart name prefix.

* prediction -> dimensions chart

* reorder funcs

* Split charts across types with correct priority

* Ignore training request when chart is under replication.

* Track global number of models consulted.

* Cleanup config.

* initial readme updates

* fix readme

* readme

* Fix function definition when ML is disabled.

* Add dummy ml_chart_update_{begin,end}

* Remove profile_plugin

* Define chart priorities under collectors/all.h

* s/curr_t/current_time/

Co-authored-by: Andrew Maguire <andrewm4894@gmail.com>

3 files changed, 35 insertions, 5 deletions

diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index a4e9d321ff..e369fb2b87 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -52,6 +52,7 @@ static struct global_statistics {
     uint64_t ml_queries_made;
     uint64_t ml_db_points_read;
     uint64_t ml_result_points_generated;
+    uint64_t ml_models_consulted;
 
     uint64_t exporters_queries_made;
     uint64_t exporters_db_points_read;
@@ -88,6 +89,10 @@ void global_statistics_ml_query_completed(size_t points_read) {
     __atomic_fetch_add(&global_statistics.ml_db_points_read, points_read, __ATOMIC_RELAXED);
 }
 
+void global_statistics_ml_models_consulted(size_t models_consulted) {
+    __atomic_fetch_add(&global_statistics.ml_models_consulted, models_consulted, __ATOMIC_RELAXED);
+}
+
 void global_statistics_exporters_query_completed(size_t points_read) {
     __atomic_fetch_add(&global_statistics.exporters_queries_made, 1, __ATOMIC_RELAXED);
     __atomic_fetch_add(&global_statistics.exporters_db_points_read, points_read, __ATOMIC_RELAXED);
@@ -193,6 +198,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
     gs->ml_queries_made              = __atomic_load_n(&global_statistics.ml_queries_made, __ATOMIC_RELAXED);
     gs->ml_db_points_read            = __atomic_load_n(&global_statistics.ml_db_points_read, __ATOMIC_RELAXED);
     gs->ml_result_points_generated   = __atomic_load_n(&global_statistics.ml_result_points_generated, __ATOMIC_RELAXED);
+    gs->ml_models_consulted          = __atomic_load_n(&global_statistics.ml_models_consulted, __ATOMIC_RELAXED);
 
     gs->exporters_queries_made       = __atomic_load_n(&global_statistics.exporters_queries_made, __ATOMIC_RELAXED);
     gs->exporters_db_points_read     = __atomic_load_n(&global_statistics.exporters_db_points_read, __ATOMIC_RELAXED);
@@ -653,6 +659,34 @@ static void global_statistics_charts(void) {
 
         rrdset_done(st_points_stored);
     }
+
+    {
+        static RRDSET *st = NULL;
+        static RRDDIM *rd = NULL;
+
+        if (unlikely(!st)) {
+            st = rrdset_create_localhost(
+                    "netdata" // type
+                    , "ml_models_consulted" // id
+                    , NULL // name
+                    , "ml" // family
+                    , NULL // context
+                    , "KMeans models used for prediction" // title
+                    , "models" // units
+                    , "netdata" // plugin
+                    , "ml" // module
+                    , 131004 // priority
+                    , localhost->rrd_update_every // update_every
+                    , RRDSET_TYPE_STACKED // chart_type
+            );
+
+            rd = rrddim_add(st, "num_models_consulted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+        }
+
+        rrddim_set_by_pointer(st, rd, (collected_number) gs.ml_models_consulted);
+
+        rrdset_done(st);
+    }
 }
 
 // ----------------------------------------------------------------------------
diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h
index f7d6775cf5..c7f53781aa 100644
--- a/daemon/global_statistics.h
+++ b/daemon/global_statistics.h
@@ -9,6 +9,7 @@
 // global statistics
 
 void global_statistics_ml_query_completed(size_t points_read);
+void global_statistics_ml_models_consulted(size_t models_consulted);
 void global_statistics_exporters_query_completed(size_t points_read);
 void global_statistics_backfill_query_completed(size_t points_read);
 void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source);
diff --git a/daemon/main.c b/daemon/main.c
index 97f3e85fa6..941412697f 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -1027,11 +1027,6 @@ int main(int argc, char **argv) {
                         else if(strcmp(optarg, "escapetest") == 0) {
                             return command_argument_sanitization_tests();
                         }
-#ifdef ENABLE_ML_TESTS
-                        else if(strcmp(optarg, "mltest") == 0) {
-                            return test_ml(argc, argv);
-                        }
-#endif
 #ifdef ENABLE_DBENGINE
                         else if(strcmp(optarg, "mctest") == 0) {
                             unittest_running = true;