/api/v2/contexts (#14592)

* preparation for /api/v2/contexts

* working /api/v2/contexts

* add anomaly rate information in all statistics; when sum-count is requested, return sums and counts instead of averages

* minor fix

* query targegt now accurately counts hosts, contexts, instances, dimensions, metrics

* cleanup /api/v2/contexts

* full text search with /api/v2/contexts

* simple patterns now support the option to search ignoring case

* full text search API with /api/v2/q

* simple pattern execution optimization

* do not show q when not given

* full text search accounting

* separated /api/v2/nodes from /api/v2/contexts

* fix ssv queries for group_by

* count query instances queried and failed per context and host

* split rrdcontext.c to multiple files

* add query totals

* fix anomaly rate calculation; provide "ni" for indexing hosts

* do not generate zero valued members

* faster calculation of anomaly rate; by just summing integers for each db points and doing math once for every generated point

* fix typo when printing dimensions totals

* added option minify to remove spaces and newlines fron JSON output

* send instance ids and names when they differ

* do not add in query target dimensions, instances, contexts and hosts for which there is no retention in the current timeframe

* fix for the previous + renames and code cleanup

* when a dimension is filtered, include in the response all the other dimensions that are selectable

* do not add nodes that do not have retention in the current window

* move selection of dimensions to query_dimension_add(), instead of query_metric_add()

* increase the pre-processing capacity of queries

* generate instance fqdn ids and names only when they are needed

* provide detailed statistics about tiers retention, queries, points, update_every

* late allocation of query dimensions

* cleanup

* more cleanup

* support for annotations per displayed point, RESET and PARTIAL

* new type annotations

* if a chart is not linked to contexts and it is collected, link it when it is collected

* make ML run reentrant

* make ML rrdr query synchronous

* optimize replication memory allocation of replication_sort_entry

* change units to percentage, when requesting a coefficinet of variation, or a percentage query

* initialize replication before starting main threads

* properly decrement no room requests counter

* propagate the non-zero flag to group-by

* the same by avoiding the extra loop

* respect non-zero in all dimension arrays

* remove dictionary garbage collection from dictionary_entries() and dictionary_version()

* be more verbose when jv2 indexing is postponed

* prevent infinite loop

* use hidden dimensions even when dimensions pattern is unset

* traverse hosts using dictionaries

* fix dictionary unittests

3 files changed, 7 insertions, 8 deletions

diff --git a/ml/Config.cc b/ml/Config.cc
index d7ddff800d..8a31f2bd6b 100644
--- a/ml/Config.cc
+++ b/ml/Config.cc
@@ -101,12 +101,12 @@ void nml_config_load(nml_config_t *cfg) {
     cfg->dimension_anomaly_score_threshold = dimension_anomaly_rate_threshold;
 
     cfg->hosts_to_skip = config_get(config_section_ml, "hosts to skip from training", "!*");
-    cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
+    cfg->sp_host_to_skip = simple_pattern_create(cfg->hosts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
 
     // Always exclude anomaly_detection charts from training.
     cfg->charts_to_skip = "anomaly_detection.* ";
     cfg->charts_to_skip += config_get(config_section_ml, "charts to skip from training", "netdata.*");
-    cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
+    cfg->sp_charts_to_skip = simple_pattern_create(cfg->charts_to_skip.c_str(), NULL, SIMPLE_PATTERN_EXACT, true);
 
     cfg->stream_anomaly_detection_charts = config_get_boolean(config_section_ml, "stream anomaly detection charts", true);
 }
diff --git a/ml/ad_charts.cc b/ml/ad_charts.cc
index 47dd97df02..d1607bf70f 100644
--- a/ml/ad_charts.cc
+++ b/ml/ad_charts.cc
@@ -276,7 +276,7 @@ void nml_update_host_and_detection_rate_charts(nml_host_t *host, collected_numbe
                 0, /* timeout */
                 0, /* tier */
                 QUERY_SOURCE_ML,
-                STORAGE_PRIORITY_BEST_EFFORT
+                STORAGE_PRIORITY_SYNCHRONOUS
         );
 
         if (R) {
diff --git a/ml/nml.cc b/ml/nml.cc
index 33be2f4b38..1fc7a6597a 100644
--- a/ml/nml.cc
+++ b/ml/nml.cc
@@ -1118,17 +1118,16 @@ void *nml_detect_main(void *arg) {
         worker_is_idle();
         heartbeat_next(&hb, USEC_PER_SEC);
 
-        rrd_rdlock();
+        void *rhp;
+        dfe_start_reentrant(rrdhost_root_index, rhp) {
+            RRDHOST *rh = (RRDHOST *) rhp;
 
-        RRDHOST *rh;
-        rrdhost_foreach_read(rh) {
             if (!rh->ml_host)
                 continue;
 
             nml_host_detect_once(reinterpret_cast<nml_host_t *>(rh->ml_host));
         }
-
-        rrd_unlock();
+        dfe_done(rhp);
     }
 
     return NULL;