/api/v2/contexts (#14592)

* preparation for /api/v2/contexts

* working /api/v2/contexts

* add anomaly rate information in all statistics; when sum-count is requested, return sums and counts instead of averages

* minor fix

* query targegt now accurately counts hosts, contexts, instances, dimensions, metrics

* cleanup /api/v2/contexts

* full text search with /api/v2/contexts

* simple patterns now support the option to search ignoring case

* full text search API with /api/v2/q

* simple pattern execution optimization

* do not show q when not given

* full text search accounting

* separated /api/v2/nodes from /api/v2/contexts

* fix ssv queries for group_by

* count query instances queried and failed per context and host

* split rrdcontext.c to multiple files

* add query totals

* fix anomaly rate calculation; provide "ni" for indexing hosts

* do not generate zero valued members

* faster calculation of anomaly rate; by just summing integers for each db points and doing math once for every generated point

* fix typo when printing dimensions totals

* added option minify to remove spaces and newlines fron JSON output

* send instance ids and names when they differ

* do not add in query target dimensions, instances, contexts and hosts for which there is no retention in the current timeframe

* fix for the previous + renames and code cleanup

* when a dimension is filtered, include in the response all the other dimensions that are selectable

* do not add nodes that do not have retention in the current window

* move selection of dimensions to query_dimension_add(), instead of query_metric_add()

* increase the pre-processing capacity of queries

* generate instance fqdn ids and names only when they are needed

* provide detailed statistics about tiers retention, queries, points, update_every

* late allocation of query dimensions

* cleanup

* more cleanup

* support for annotations per displayed point, RESET and PARTIAL

* new type annotations

* if a chart is not linked to contexts and it is collected, link it when it is collected

* make ML run reentrant

* make ML rrdr query synchronous

* optimize replication memory allocation of replication_sort_entry

* change units to percentage, when requesting a coefficinet of variation, or a percentage query

* initialize replication before starting main threads

* properly decrement no room requests counter

* propagate the non-zero flag to group-by

* the same by avoiding the extra loop

* respect non-zero in all dimension arrays

* remove dictionary garbage collection from dictionary_entries() and dictionary_version()

* be more verbose when jv2 indexing is postponed

* prevent infinite loop

* use hidden dimensions even when dimensions pattern is unset

* traverse hosts using dictionaries

* fix dictionary unittests

1 files changed, 20 insertions, 14 deletions

diff --git a/daemon/main.c b/daemon/main.c
index d39a76da3e..04bb9d377a 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -530,38 +530,41 @@ void web_server_config_options(void)
         web_x_frame_options = NULL;
 
     web_allow_connections_from =
-        simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"),
-                              NULL, SIMPLE_PATTERN_EXACT);
+            simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"),
+                                  NULL, SIMPLE_PATTERN_EXACT, true);
     web_allow_connections_dns  =
         make_dns_decision(CONFIG_SECTION_WEB, "allow connections by dns", "heuristic", web_allow_connections_from);
     web_allow_dashboard_from   =
-        simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"),
-                              NULL, SIMPLE_PATTERN_EXACT);
+            simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"),
+                                  NULL, SIMPLE_PATTERN_EXACT, true);
     web_allow_dashboard_dns    =
         make_dns_decision(CONFIG_SECTION_WEB, "allow dashboard by dns", "heuristic", web_allow_dashboard_from);
     web_allow_badges_from      =
-        simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT);
+            simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT,
+                                  true);
     web_allow_badges_dns       =
         make_dns_decision(CONFIG_SECTION_WEB, "allow badges by dns", "heuristic", web_allow_badges_from);
     web_allow_registry_from    =
-        simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT);
+            simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT,
+                                  true);
     web_allow_registry_dns     = make_dns_decision(CONFIG_SECTION_REGISTRY, "allow by dns", "heuristic",
                                                    web_allow_registry_from);
     web_allow_streaming_from   = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"),
-                                                       NULL, SIMPLE_PATTERN_EXACT);
+                                                       NULL, SIMPLE_PATTERN_EXACT, true);
     web_allow_streaming_dns    = make_dns_decision(CONFIG_SECTION_WEB, "allow streaming by dns", "heuristic",
                                                    web_allow_streaming_from);
     // Note the default is not heuristic, the wildcards could match DNS but the intent is ip-addresses.
     web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from",
-                                                       "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*"
-                                                       " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*"
-                                                       " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*"
-                                                       " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT);
+                                                                  "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*"
+                                                                  " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*"
+                                                                  " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*"
+                                                                  " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT,
+                                                       true);
     web_allow_netdataconf_dns  =
         make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_netdataconf_from);
     web_allow_mgmt_from        =
-        simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"),
-                              NULL, SIMPLE_PATTERN_EXACT);
+            simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"),
+                                  NULL, SIMPLE_PATTERN_EXACT, true);
     web_allow_mgmt_dns         =
         make_dns_decision(CONFIG_SECTION_WEB, "allow management by dns","heuristic",web_allow_mgmt_from);
 
@@ -1325,6 +1328,7 @@ int pgc_unittest(void);
 int mrg_unittest(void);
 int julytest(void);
 int pluginsd_parser_unittest(void);
+void replication_initialize(void);
 
 int main(int argc, char **argv) {
     // initialize the system clocks
@@ -1612,7 +1616,7 @@ int main(int argc, char **argv) {
                             size_t len = strlen(needle) + 1;
                             char wildcarded[len];
 
-                            SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT);
+                            SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT, true);
                             int ret = simple_pattern_matches_extract(p, needle, wildcarded, len);
                             simple_pattern_free(p);
 
@@ -1897,6 +1901,8 @@ int main(int argc, char **argv) {
 
         get_system_timezone();
 
+        replication_initialize();
+
         // --------------------------------------------------------------------
         // get the certificate and start security