/api/v2/X improvements part 3 (#14665)

* max web request size to 64KB

* fix the request too big message

* increase max request reading tries to 100

* support for bigger web requests

* add "avg" as a shortcut for "average" to both group by aggregation and time aggregation; discard the last partial points of a query in play mode, up to max update every; group by hidden dimensions too

* better implementation for partial data trimming

* added group_by=selected to return only one dimension for all selected metrics

* fix acceptance of group_by=selected

* passing option "raw" disables partial data trimming

* remove obsolete option "plan"; use "debug"

* fix view.min and view.max calculation - there were 2 bugs: a) min and max were reset for every row and b) min and max were corrupted by GBC and AR printing

* per row annotations

* added time column to point annotations

* disable caching for /api/v2/contexts responses

* added api format json2 that returns an array for each points, having all the point values and annotations in them

* work on swagger about /api/v2

* prevent infinite loop

* cleanup and swagger work

* allow negative simple pattern expressions to work as expected

* do not lookup in the dictionary empty names

* garbage collect dictionaries

* make query_target allocate less aggressively; queries fill the remaining points with nulls

* reusable query ops to save memory on huge queries

* move parts of query plans into query ops to save query target memory

* remove storage engine from query metric tiers, to save memory, and recalculate it when it is needed

5 files changed, 44 insertions, 21 deletions

diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index 25ea0d188f..1cf1ec981d 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -732,6 +732,15 @@ static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
     wb->json.stack[wb->json.depth].count++;
 }
 
+static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
+    if(wb->json.stack[wb->json.depth].count)
+        buffer_fast_strcat(wb, ",", 1);
+
+    buffer_print_int64(wb, value);
+    buffer_fast_strcat(wb, "000", 3);
+    wb->json.stack[wb->json.depth].count++;
+}
+
 static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
     if(wb->json.stack[wb->json.depth].count)
         buffer_fast_strcat(wb, ",", 1);
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
index e4128d01eb..d161de621f 100644
--- a/libnetdata/dictionary/dictionary.c
+++ b/libnetdata/dictionary/dictionary.c
@@ -811,6 +811,10 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
                           examined, deleted, pending);
 }
 
+void dictionary_garbage_collect(DICTIONARY *dict) {
+    garbage_collect_pending_deletes(dict);
+}
+
 // ----------------------------------------------------------------------------
 // reference counters
 
@@ -2149,6 +2153,8 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(D
     if(unlikely(is_master_dictionary(dict)))
         fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries.");
 
+    garbage_collect_pending_deletes(dict);
+
     dictionary_acquired_item_dup(dict->master, master_item);
     DICTIONARY_ITEM *item = dict_item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
     dictionary_acquired_item_release(dict->master, master_item);
diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h
index c74f17d354..9dd054caba 100644
--- a/libnetdata/dictionary/dictionary.h
+++ b/libnetdata/dictionary/dictionary.h
@@ -156,6 +156,8 @@ void dictionary_flush(DICTIONARY *dict);
 
 void dictionary_version_increment(DICTIONARY *dict);
 
+void dictionary_garbage_collect(DICTIONARY *dict);
+
 // ----------------------------------------------------------------------------
 // Set an item in the dictionary
 //
diff --git a/libnetdata/simple_pattern/simple_pattern.c b/libnetdata/simple_pattern/simple_pattern.c
index 39b9707619..a26ae4f920 100644
--- a/libnetdata/simple_pattern/simple_pattern.c
+++ b/libnetdata/simple_pattern/simple_pattern.c
@@ -265,7 +265,7 @@ static inline bool match_pattern(struct simple_pattern *m, const char *str, size
     return false;
 }
 
-static inline int simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
+static inline SIMPLE_PATTERN_RESULT simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
     struct simple_pattern *m, *root = (struct simple_pattern *)list;
 
     for(m = root; m ; m = m->next) {
@@ -274,31 +274,31 @@ static inline int simple_pattern_matches_extract_with_length(SIMPLE_PATTERN *lis
         if(unlikely(ws)) *ws = '\0';
 
         if (match_pattern(m, str, len, ws, &wss)) {
-            if (m->negative) return 0;
-            return 1;
+            if (m->negative) return SP_MATCHED_NEGATIVE;
+            return SP_MATCHED_POSITIVE;
         }
     }
 
-    return 0;
+    return SP_NOT_MATCHED;
 }
 
-int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size) {
-    if(!list || !str || buffer_strlen(str)) return 0;
+SIMPLE_PATTERN_RESULT simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size) {
+    if(!list || !str || buffer_strlen(str)) return SP_NOT_MATCHED;
     return simple_pattern_matches_extract_with_length(list, buffer_tostring(str), buffer_strlen(str), wildcarded, wildcarded_size);
 }
 
-int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, STRING *str, char *wildcarded, size_t wildcarded_size) {
-    if(!list || !str) return 0;
+SIMPLE_PATTERN_RESULT simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, STRING *str, char *wildcarded, size_t wildcarded_size) {
+    if(!list || !str) return SP_NOT_MATCHED;
     return simple_pattern_matches_extract_with_length(list, string2str(str), string_strlen(str), wildcarded, wildcarded_size);
 }
 
-int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
-    if(!list || !str || !*str) return 0;
+SIMPLE_PATTERN_RESULT simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
+    if(!list || !str || !*str) return SP_NOT_MATCHED;
     return simple_pattern_matches_extract_with_length(list, str, strlen(str), wildcarded, wildcarded_size);
 }
 
-int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
-    if(!list || !str || !*str || !len) return 0;
+SIMPLE_PATTERN_RESULT simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size) {
+    if(!list || !str || !*str || !len) return SP_NOT_MATCHED;
     return simple_pattern_matches_extract_with_length(list, str, len, wildcarded, wildcarded_size);
 }
 
diff --git a/libnetdata/simple_pattern/simple_pattern.h b/libnetdata/simple_pattern/simple_pattern.h
index 135ac3b7c8..1a8d8f7d66 100644
--- a/libnetdata/simple_pattern/simple_pattern.h
+++ b/libnetdata/simple_pattern/simple_pattern.h
@@ -13,6 +13,12 @@ typedef enum __attribute__ ((__packed__)) {
     SIMPLE_PATTERN_SUBSTRING
 } SIMPLE_PREFIX_MODE;
 
+typedef enum __attribute__ ((__packed__)) {
+    SP_NOT_MATCHED,
+    SP_MATCHED_NEGATIVE,
+    SP_MATCHED_POSITIVE,
+} SIMPLE_PATTERN_RESULT;
+
 typedef void SIMPLE_PATTERN;
 
 // create a simple_pattern from the string given
@@ -20,18 +26,18 @@ typedef void SIMPLE_PATTERN;
 // should be considered PREFIX matches.
 SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode, bool case_sensitive);
 
-// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
-int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
-
 struct netdata_string;
-int simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, struct netdata_string *str, char *wildcarded, size_t wildcarded_size);
-int simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size);
-int simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size);
+
+// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
+SIMPLE_PATTERN_RESULT simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
+SIMPLE_PATTERN_RESULT simple_pattern_matches_string_extract(SIMPLE_PATTERN *list, struct netdata_string *str, char *wildcarded, size_t wildcarded_size);
+SIMPLE_PATTERN_RESULT simple_pattern_matches_buffer_extract(SIMPLE_PATTERN *list, BUFFER *str, char *wildcarded, size_t wildcarded_size);
+SIMPLE_PATTERN_RESULT simple_pattern_matches_length_extract(SIMPLE_PATTERN *list, const char *str, size_t len, char *wildcarded, size_t wildcarded_size);
 
 // test if string str is matched from the pattern
-#define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0)
-#define simple_pattern_matches_string(list, str) simple_pattern_matches_string_extract(list, str, NULL, 0)
-#define simple_pattern_matches_buffer(list, str) simple_pattern_matches_buffer_extract(list, str, NULL, 0)
+#define simple_pattern_matches(list, str) (simple_pattern_matches_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
+#define simple_pattern_matches_string(list, str) (simple_pattern_matches_string_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
+#define simple_pattern_matches_buffer(list, str) (simple_pattern_matches_buffer_extract(list, str, NULL, 0) == SP_MATCHED_POSITIVE)
 
 // free a simple_pattern that was created with simple_pattern_create()
 // list can be NULL, in which case, this does nothing.