/api/v2/X part 5 (#14718)

* query timestamps are now pre-determined and alignment on timestamps is guarranteed

* turn internal_fatal() to internal_error() to investigate the issue

* handle query when no data exist in the db

* check for non NULL dict when running dictionary garbage collect

* support API v2 requests via ACLK

* add nodes detailed information to /api/v2/nodes

* fixed keys and added dummy nodes for completeness

* added nodes_hard_hash, alerts_hard_hash, alerts_soft_hash; started building a nodes status object to reflect the current status of a node

* make sure replication does not double count charts that are already being replicated

* expose min and max in sts structures

* added view_minimum_value and view_maximum_value; percentage calculation is now an additional pass on the data, removed from formatters; absolute value calculation is now done at the query level, removed from formatters

* respect trimming in percentage calculation; updated swagger

* api/v2/weights preparative work to support multi-node queries - still single node though

* multi-node /api/v2/weights endpoint, supporting all the filtering parameters of /api/v2/data

* when passing the raw option, the query exposes the hidden dimensions

* fix compilation issues on older systems

* the query engine now calculates per dimension min, max, sum, count, anomaly count

* use the macro to calculate storage point anomaly rate

* weights endpoint exposing version hashes

* weights method=value shows min, max, average, sum, count, anomaly count, anomaly rate

* query: expose RESET flag; do not add the same point multiple times to the aggregated point

* weights: more compact output

* weights requests can be interrupted

* all /api/v2 requests can be interrupted and timeout

* allow relative timestamps in weights

* fix macos compilation warnings

* Revert "fix macos compilation warnings"

This reverts commit 8a1d24e41e9b58de566ac59f0c4b1c465bcc0592.

* /api/v2/data group-by now works on dimension names, not ids

* /api/v2/weights does not query metrics without retention and new output format

* /api/v2/weights value and anomaly queries do context queries when contexts are filtered; query timeout is now always in ms

4 files changed, 152 insertions, 8 deletions

diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index d19aad81e8..73aa428bd9 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -755,6 +755,14 @@ static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE
     wb->json.stack[wb->json.depth].count++;
 }
 
+static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {
+    if(wb->json.stack[wb->json.depth].count)
+        buffer_fast_strcat(wb, ",", 1);
+
+    buffer_print_int64(wb, value);
+    wb->json.stack[wb->json.depth].count++;
+}
+
 static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {
     if(wb->json.stack[wb->json.depth].count)
         buffer_fast_strcat(wb, ",", 1);
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
index d161de621f..42e4a99f14 100644
--- a/libnetdata/dictionary/dictionary.c
+++ b/libnetdata/dictionary/dictionary.c
@@ -812,6 +812,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
 }
 
 void dictionary_garbage_collect(DICTIONARY *dict) {
+    if(!dict) return;
     garbage_collect_pending_deletes(dict);
 }
 
@@ -2277,7 +2278,7 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
     dfe->counter = 0;
     dfe->dict = dict;
     dfe->rw = rw;
-
+    dfe->locked = true;
     ll_recursive_lock(dict, dfe->rw);
 
     DICTIONARY_STATS_TRAVERSALS_PLUS1(dict);
@@ -2300,8 +2301,10 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
         dfe->value = NULL;
     }
 
-    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
         ll_recursive_unlock(dfe->dict, dfe->rw);
+        dfe->locked = false;
+    }
 
     return dfe->value;
 }
@@ -2317,8 +2320,10 @@ void *dictionary_foreach_next(DICTFE *dfe) {
         return NULL;
     }
 
-    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT) || !dfe->locked) {
         ll_recursive_lock(dfe->dict, dfe->rw);
+        dfe->locked = true;
+    }
 
     // the item we just did
     DICTIONARY_ITEM *item = dfe->item;
@@ -2348,12 +2353,21 @@ void *dictionary_foreach_next(DICTFE *dfe) {
         dfe->value = NULL;
     }
 
-    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+    if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
         ll_recursive_unlock(dfe->dict, dfe->rw);
+        dfe->locked = false;
+    }
 
     return dfe->value;
 }
 
+void dictionary_foreach_unlock(DICTFE *dfe) {
+    if(dfe->locked) {
+        ll_recursive_unlock(dfe->dict, dfe->rw);
+        dfe->locked = false;
+    }
+}
+
 void dictionary_foreach_done(DICTFE *dfe) {
     if(unlikely(!dfe || !dfe->dict)) return;
 
@@ -2371,8 +2385,10 @@ void dictionary_foreach_done(DICTFE *dfe) {
         // item_release(dfe->dict, item);
     }
 
-    if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT))
+    if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT) && dfe->locked) {
         ll_recursive_unlock(dfe->dict, dfe->rw);
+        dfe->locked = false;
+    }
 
     dfe->dict = NULL;
     dfe->item = NULL;
diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h
index 9dd054caba..c13d784cb2 100644
--- a/libnetdata/dictionary/dictionary.h
+++ b/libnetdata/dictionary/dictionary.h
@@ -263,19 +263,20 @@ void dictionary_write_lock(DICTIONARY *dict);
 void dictionary_write_unlock(DICTIONARY *dict);
 
 typedef DICTFE_CONST struct dictionary_foreach {
-    DICTIONARY *dict;           // the dictionary upon we work
+    DICTIONARY *dict; // the dictionary upon we work
 
     DICTIONARY_ITEM *item;      // the item we work on, to remember the position we are at
                                 // this can be used with dictionary_acquired_item_dup() to
                                 // acquire the currently working item.
 
-    DICTFE_CONST char *name;    // the dictionary name of the last item used
+    const char *name;           // the dictionary name of the last item used
     void *value;                // the dictionary value of the last item used
                                 // same as the return value of dictfe_start() and dictfe_next()
 
     size_t counter;             // counts the number of iterations made, starting from zero
 
     char rw;                    // the lock mode 'r' or 'w'
+    bool locked;                // true when the dictionary is locked
 } DICTFE;
 
 #define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ)
@@ -296,9 +297,12 @@ typedef DICTFE_CONST struct dictionary_foreach {
             dictionary_foreach_done(&value ## _dfe);                                                \
         } while(0)
 
+#define dfe_unlock(value) dictionary_foreach_unlock(&value ## _dfe);
+
 void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw);
 void *dictionary_foreach_next(DICTFE *dfe);
 void  dictionary_foreach_done(DICTFE *dfe);
+void  dictionary_foreach_unlock(DICTFE *dfe);
 
 // ----------------------------------------------------------------------------
 // Get statistics about the dictionary
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index da2d6c43ef..31902b3966 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -366,6 +366,123 @@ size_t judy_aral_structures(void);
 
 // ---------------------------------------------------------------------------------------------
 
+#include "storage_number/storage_number.h"
+
+typedef struct storage_point {
+    NETDATA_DOUBLE min;     // when count > 1, this is the minimum among them
+    NETDATA_DOUBLE max;     // when count > 1, this is the maximum among them
+    NETDATA_DOUBLE sum;     // the point sum - divided by count gives the average
+
+    // end_time - start_time = point duration
+    time_t start_time_s;    // the time the point starts
+    time_t end_time_s;      // the time the point ends
+
+    size_t count;           // the number of original points aggregated
+    size_t anomaly_count;   // the number of original points found anomalous
+
+    SN_FLAGS flags;         // flags stored with the point
+} STORAGE_POINT;
+
+#define storage_point_unset(x)                     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 0;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time_s = 0;                               \
+    (x).end_time_s = 0;                                 \
+    } while(0)
+
+#define storage_point_empty(x, start_s, end_s)     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 1;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time_s = start_s;                         \
+    (x).end_time_s = end_s;                             \
+    } while(0)
+
+#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
+
+#define storage_point_is_unset(x) (!(x).count)
+#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
+
+#define storage_point_merge_to(dst, src) do {           \
+        if(storage_point_is_unset(dst))                 \
+            (dst) = (src);                              \
+                                                        \
+        else if(!storage_point_is_unset(src) &&         \
+                !storage_point_is_gap(src)) {           \
+                                                        \
+            if((src).start_time_s < (dst).start_time_s) \
+                (dst).start_time_s = (src).start_time_s;\
+                                                        \
+            if((src).end_time_s > (dst).end_time_s)     \
+                (dst).end_time_s = (src).end_time_s;    \
+                                                        \
+            if((src).min < (dst).min)                   \
+                (dst).min = (src).min;                  \
+                                                        \
+            if((src).max > (dst).max)                   \
+                (dst).max = (src).max;                  \
+                                                        \
+            (dst).sum += (src).sum;                     \
+                                                        \
+            (dst).count += (src).count;                 \
+            (dst).anomaly_count += (src).anomaly_count; \
+                                                        \
+            (dst).flags |= (src).flags & SN_FLAG_RESET; \
+        }                                               \
+} while(0)
+
+#define storage_point_add_to(dst, src) do {             \
+        if(storage_point_is_unset(dst))                 \
+            (dst) = (src);                              \
+                                                        \
+        else if(!storage_point_is_unset(src) &&         \
+                !storage_point_is_gap(src)) {           \
+                                                        \
+            if((src).start_time_s < (dst).start_time_s) \
+                (dst).start_time_s = (src).start_time_s;\
+                                                        \
+            if((src).end_time_s > (dst).end_time_s)     \
+                (dst).end_time_s = (src).end_time_s;    \
+                                                        \
+            (dst).min += (src).min;                     \
+            (dst).max += (src).max;                     \
+            (dst).sum += (src).sum;                     \
+                                                        \
+            (dst).count += (src).count;                 \
+            (dst).anomaly_count += (src).anomaly_count; \
+                                                        \
+            (dst).flags |= (src).flags & SN_FLAG_RESET; \
+        }                                               \
+} while(0)
+
+#define storage_point_make_positive(sp) do {            \
+        if(!storage_point_is_unset(sp) &&               \
+           !storage_point_is_gap(sp)) {                 \
+                                                        \
+            if(unlikely(signbit((sp).sum)))             \
+                (sp).sum = -(sp).sum;                   \
+                                                        \
+            if(unlikely(signbit((sp).min)))             \
+                (sp).min = -(sp).min;                   \
+                                                        \
+            if(unlikely(signbit((sp).max)))             \
+                (sp).max = -(sp).max;                   \
+                                                        \
+            if(unlikely((sp).min > (sp).max)) {         \
+                NETDATA_DOUBLE t = (sp).min;            \
+                (sp).min = (sp).max;                    \
+                (sp).max = t;                           \
+            }                                           \
+        }                                               \
+} while(0)
+
+#define storage_point_anomaly_rate(sp) \
+    (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
+
+// ---------------------------------------------------------------------------------------------
 
 void netdata_fix_chart_id(char *s);
 void netdata_fix_chart_name(char *s);
@@ -517,7 +634,6 @@ extern char *netdata_configured_host_prefix;
 #include "libjudy/src/Judy.h"
 #include "july/july.h"
 #include "os.h"
-#include "storage_number/storage_number.h"
 #include "threads/threads.h"
 #include "buffer/buffer.h"
 #include "locks/locks.h"