modularize the query api (#4443)

* modularized exporters

* modularized API data queries

* optimized queries

* modularized API data reduction methods

* modularized api queries

* added new directories in makefiles

* added median db query

* moved all RRDR_GROUPING related to query.h

* added stddev query

* operational median and stddev

* working simple exponential smoothing

* too complex to do it right

* fixed ses

* fixed ses

* rewrote query engine

* fix double-exponential-smoothing

* cleanup

* fixed bug identified by @vlvkobal at rrdset_first_slot()

* enable freeipmi on systems with libipmimonitoring; #4440

5 files changed, 82 insertions, 24 deletions

diff --git a/database/rrd.h b/database/rrd.h
index 57d94c4c86..19eb100cd4 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -15,7 +15,7 @@ typedef struct rrdcalctemplate RRDCALCTEMPLATE;
 typedef struct alarm_entry ALARM_ENTRY;
 
 #include "../daemon/common.h"
-
+#include "web/api/queries/query.h"
 #include "rrdvar.h"
 #include "rrdsetvar.h"
 #include "rrddimvar.h"
@@ -749,28 +749,84 @@ extern void rrdset_isnot_obsolete(RRDSET *st);
 #define rrdset_first_entry_t(st) ((time_t)(rrdset_last_entry_t(st) - rrdset_duration(st)))
 
 // get the last slot updated in the round robin database
-#define rrdset_last_slot(st) ((unsigned long)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
+#define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
 
 // get the first / oldest slot updated in the round robin database
-#define rrdset_first_slot(st) ((unsigned long)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
+// #define rrdset_first_slot(st) ((size_t)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
+
+// return the slot that has the oldest value
+
+static inline size_t rrdset_first_slot(RRDSET *st) {
+    if(st->counter >= (size_t)st->entries) {
+        // the database has been rotated at least once
+        // the oldest entry is the one that will be next
+        // overwritten by data collection
+        return (size_t)st->current_entry;
+    }
+
+    // we do not have rotated the db yet
+    // so 0 is the first entry
+    return 0;
+}
 
 // get the slot of the round robin database, for the given timestamp (t)
 // it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
-#define rrdset_time2slot(st, t) ( \
-        (  (time_t)(t) >= rrdset_last_entry_t(st))  ? ( rrdset_last_slot(st) ) : \
-        ( ((time_t)(t) <= rrdset_first_entry_t(st)) ?   rrdset_first_slot(st) : \
-        ( (rrdset_last_slot(st) >= (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) ? \
-          (rrdset_last_slot(st) -  (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) : \
-          (rrdset_last_slot(st) -  (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) + (unsigned long)(st)->entries ) \
-        )))
+static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {
+    size_t ret = 0;
+
+    if(t >= rrdset_last_entry_t(st)) {
+        // the requested time is after the last entry we have
+        ret = rrdset_last_slot(st);
+    }
+    else {
+        if(t <= rrdset_first_entry_t(st)) {
+            // the requested time is before the first entry we have
+            ret = rrdset_first_slot(st);
+        }
+        else {
+            if(rrdset_last_slot(st) >= ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every)))
+                ret = rrdset_last_slot(st) - ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every));
+            else
+                ret = rrdset_last_slot(st) - ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every)) + (unsigned long)st->entries;
+        }
+    }
+
+    if(unlikely(ret >= (size_t)st->entries)) {
+        error("INTERNAL ERROR: rrdset_time2slot() on %s returns values outside entries", st->name);
+        ret = (size_t)(st->entries - 1);
+    }
+
+    return ret;
+}
 
 // get the timestamp of a specific slot in the round robin database
-#define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \
-        ((unsigned long)(st)->update_every * ( \
-                ( (unsigned long)(slot) > rrdset_last_slot(st)) ? \
-                ( (rrdset_last_slot(st) - (unsigned long)(slot) + (unsigned long)(st)->entries) ) : \
-                ( (rrdset_last_slot(st) - (unsigned long)(slot)) )) \
-        ))
+static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
+    time_t ret;
+
+    if(slot >= (size_t)st->entries) {
+        error("INTERNAL ERROR: caller of rrdset_slot2time() gives invalid slot %zu", slot);
+        slot = (size_t)st->entries - 1;
+    }
+
+    if(slot > rrdset_last_slot(st)) {
+        ret = rrdset_last_entry_t(st) - (size_t)st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries);
+    }
+    else {
+        ret = rrdset_last_entry_t(st) - (size_t)st->update_every;
+    }
+
+    if(unlikely(ret < rrdset_first_entry_t(st))) {
+        error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);
+        ret = rrdset_first_entry_t(st);
+    }
+
+    if(unlikely(ret > rrdset_last_entry_t(st))) {
+        error("INTERNAL ERROR: rrdset_slot2time() on %s returns time into the future", st->name);
+        ret = rrdset_last_entry_t(st);
+    }
+
+    return ret;
+}
 
 // ----------------------------------------------------------------------------
 // RRD DIMENSION functions
diff --git a/database/rrdcalc.c b/database/rrdcalc.c
index 32c244ff85..7f6a896b6f 100644
--- a/database/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -363,7 +363,7 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c
             (rc->recipient)?rc->recipient:"DEFAULT",
             rc->green,
             rc->red,
-            rc->group,
+            (int)rc->group,
             rc->after,
             rc->before,
             rc->options,
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
index 3dcaf50107..0c7cd0aa1e 100644
--- a/database/rrdcalc.h
+++ b/database/rrdcalc.h
@@ -54,7 +54,7 @@ struct rrdcalc {
     // database lookup settings
 
     char *dimensions;               // the chart dimensions
-    int group;                      // grouping method: average, max, etc.
+    RRDR_GROUPING group;               // grouping method: average, max, etc.
     int before;                     // ending point in time-series
     int after;                      // starting point in time-series
     uint32_t options;               // calculation options
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 5c139fbd37..2235ecaa16 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -35,7 +35,7 @@ struct rrdcalctemplate {
     // database lookup settings
 
     char *dimensions;               // the chart dimensions
-    int group;                      // grouping method: average, max, etc.
+    RRDR_GROUPING group;               // grouping method: average, max, etc.
     int before;                     // ending point in time-series
     int after;                      // starting point in time-series
     uint32_t options;               // calculation options
diff --git a/database/rrdvar.h b/database/rrdvar.h
index 48c27cf7e6..6d1461b2aa 100644
--- a/database/rrdvar.h
+++ b/database/rrdvar.h
@@ -3,7 +3,7 @@
 #ifndef NETDATA_RRDVAR_H
 #define NETDATA_RRDVAR_H 1
 
-#include "rrd.h"
+#include "libnetdata/libnetdata.h"
 
 extern int rrdvar_compare(void *a, void *b);
 
@@ -45,6 +45,12 @@ struct rrdvar {
     time_t last_updated;
 };
 
+#define RRDVAR_MAX_LENGTH 1024
+
+extern int rrdvar_fix_name(char *variable);
+
+#include "rrd.h"
+
 extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);
 extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value);
 extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);
@@ -54,10 +60,6 @@ extern int  rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback
 
 extern calculated_number rrdvar2number(RRDVAR *rv);
 
-#define RRDVAR_MAX_LENGTH 1024
-
-extern int rrdvar_fix_name(char *variable);
-
 extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
 extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);