modularize the query api (#4443)

* modularized exporters

* modularized API data queries

* optimized queries

* modularized API data reduction methods

* modularized api queries

* added new directories in makefiles

* added median db query

* moved all RRDR_GROUPING related to query.h

* added stddev query

* operational median and stddev

* working simple exponential smoothing

* too complex to do it right

* fixed ses

* fixed ses

* rewrote query engine

* fix double-exponential-smoothing

* cleanup

* fixed bug identified by @vlvkobal at rrdset_first_slot()

* enable freeipmi on systems with libipmimonitoring; #4440

58 files changed, 4256 insertions, 1955 deletions

diff --git a/web/api/Makefile.am b/web/api/Makefile.am
index 9a16762f7c..1b9f11ec81 100644
--- a/web/api/Makefile.am
+++ b/web/api/Makefile.am
@@ -5,8 +5,14 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
 SUBDIR = \
     badges \
+    queries \
     $(NULL)
 
 dist_noinst_DATA = \
 	README.md \
 	$(NULL)
+
+dist_web_DATA = \
+	netdata-swagger.yaml \
+	netdata-swagger.json \
+	$(NULL)
diff --git a/web/api/README.md b/web/api/README.md
index 01bf8c8f96..e5c74265fd 100644
--- a/web/api/README.md
+++ b/web/api/README.md
@@ -16,68 +16,3 @@ and this [multi chart, jsfiddle example](https://jsfiddle.net/ktsaou/L5y2eqp2/):
 
 ![image](https://cloud.githubusercontent.com/assets/2662304/23824766/31a4a68c-0685-11e7-8429-8327cab64be2.png)
 
-
-## using the api from shell scripts
-
-Shell scripts can now query netdata easily:
-
-```sh
-eval "$(curl -s 'http://localhost:19999/api/v1/allmetrics')"
-```
-
-after this command, all the netdata metrics are exposed to shell. Check:
-
-```sh
-# source the metrics
-eval "$(curl -s 'http://localhost:19999/api/v1/allmetrics')"
-
-# let's see if there are variables exposed by netdata for system.cpu
-set | grep "^NETDATA_SYSTEM_CPU"
-
-NETDATA_SYSTEM_CPU_GUEST=0
-NETDATA_SYSTEM_CPU_GUEST_NICE=0
-NETDATA_SYSTEM_CPU_IDLE=95
-NETDATA_SYSTEM_CPU_IOWAIT=0
-NETDATA_SYSTEM_CPU_IRQ=0
-NETDATA_SYSTEM_CPU_NICE=0
-NETDATA_SYSTEM_CPU_SOFTIRQ=0
-NETDATA_SYSTEM_CPU_STEAL=0
-NETDATA_SYSTEM_CPU_SYSTEM=1
-NETDATA_SYSTEM_CPU_USER=4
-NETDATA_SYSTEM_CPU_VISIBLETOTAL=5
-
-# let's see the total cpu utilization of the system
-echo ${NETDATA_SYSTEM_CPU_VISIBLETOTAL}
-5
-
-# what about alarms?
-set | grep "^NETDATA_ALARM_SYSTEM_SWAP_"
-NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_STATUS=CRITICAL
-NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_VALUE=53
-NETDATA_ALARM_SYSTEM_SWAP_USED_SWAP_STATUS=CLEAR
-NETDATA_ALARM_SYSTEM_SWAP_USED_SWAP_VALUE=51
-
-# let's get the current status of the alarm 'ram in swap'
-echo ${NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_STATUS}
-CRITICAL
-
-# is it fast?
-time curl -s 'http://localhost:19999/api/v1/allmetrics' >/dev/null
-
-real  0m0,070s
-user  0m0,000s
-sys   0m0,007s
-
-# it is...
-# 0.07 seconds for curl to be loaded, connect to netdata and fetch the response back...
-```
-
-The `_VISIBLETOTAL` variable sums up all the dimensions of each chart.
-
-The format of the variables is:
-
-```sh
-NETDATA_${chart_id^^}_${dimension_id^^}="${value}"
-```
-
-The value is rounded to the closest integer, since shell script cannot process decimal numbers.
diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index b54ac0ff07..d0600359e3 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -887,3 +887,256 @@ void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const ch
         label_width + value_width / 2 -1, ceil(height - text_offset), value_escaped,
         label_width + value_width / 2 -1, ceil(height - text_offset - 1.0), value_escaped);
 }
+
+int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *url) {
+    int ret = 400;
+    buffer_flush(w->response.data);
+
+    BUFFER *dimensions = NULL;
+
+    const char *chart = NULL
+    , *before_str = NULL
+    , *after_str = NULL
+    , *points_str = NULL
+    , *multiply_str = NULL
+    , *divide_str = NULL
+    , *label = NULL
+    , *units = NULL
+    , *label_color = NULL
+    , *value_color = NULL
+    , *refresh_str = NULL
+    , *precision_str = NULL
+    , *scale_str = NULL
+    , *alarm = NULL;
+
+    int group = RRDR_GROUPING_AVERAGE;
+    uint32_t options = 0x00000000;
+
+    while(url) {
+        char *value = mystrsep(&url, "/?&");
+        if(!value || !*value) continue;
+
+        char *name = mystrsep(&value, "=");
+        if(!name || !*name) continue;
+        if(!value || !*value) continue;
+
+        debug(D_WEB_CLIENT, "%llu: API v1 badge.svg query param '%s' with value '%s'", w->id, name, value);
+
+        // name and value are now the parameters
+        // they are not null and not empty
+
+        if(!strcmp(name, "chart")) chart = value;
+        else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
+            if(!dimensions)
+                dimensions = buffer_create(100);
+
+            buffer_strcat(dimensions, "|");
+            buffer_strcat(dimensions, value);
+        }
+        else if(!strcmp(name, "after")) after_str = value;
+        else if(!strcmp(name, "before")) before_str = value;
+        else if(!strcmp(name, "points")) points_str = value;
+        else if(!strcmp(name, "group")) {
+            group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);
+        }
+        else if(!strcmp(name, "options")) {
+            options |= web_client_api_request_v1_data_options(value);
+        }
+        else if(!strcmp(name, "label")) label = value;
+        else if(!strcmp(name, "units")) units = value;
+        else if(!strcmp(name, "label_color")) label_color = value;
+        else if(!strcmp(name, "value_color")) value_color = value;
+        else if(!strcmp(name, "multiply")) multiply_str = value;
+        else if(!strcmp(name, "divide")) divide_str = value;
+        else if(!strcmp(name, "refresh")) refresh_str = value;
+        else if(!strcmp(name, "precision")) precision_str = value;
+        else if(!strcmp(name, "scale")) scale_str = value;
+        else if(!strcmp(name, "alarm")) alarm = value;
+    }
+
+    if(!chart || !*chart) {
+        buffer_no_cacheable(w->response.data);
+        buffer_sprintf(w->response.data, "No chart id is given at the request.");
+        goto cleanup;
+    }
+
+    int scale = (scale_str && *scale_str)?str2i(scale_str):100;
+
+    RRDSET *st = rrdset_find(host, chart);
+    if(!st) st = rrdset_find_byname(host, chart);
+    if(!st) {
+        buffer_no_cacheable(w->response.data);
+        buffer_svg(w->response.data, "chart not found", NAN, "", NULL, NULL, -1, scale, 0);
+        ret = 200;
+        goto cleanup;
+    }
+    st->last_accessed_time = now_realtime_sec();
+
+    RRDCALC *rc = NULL;
+    if(alarm) {
+        rc = rrdcalc_find(st, alarm);
+        if (!rc) {
+            buffer_no_cacheable(w->response.data);
+            buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1, scale, 0);
+            ret = 200;
+            goto cleanup;
+        }
+    }
+
+    long long multiply  = (multiply_str  && *multiply_str )?str2l(multiply_str):1;
+    long long divide    = (divide_str    && *divide_str   )?str2l(divide_str):1;
+    long long before    = (before_str    && *before_str   )?str2l(before_str):0;
+    long long after     = (after_str     && *after_str    )?str2l(after_str):-st->update_every;
+    int       points    = (points_str    && *points_str   )?str2i(points_str):1;
+    int       precision = (precision_str && *precision_str)?str2i(precision_str):-1;
+
+    if(!multiply) multiply = 1;
+    if(!divide) divide = 1;
+
+    int refresh = 0;
+    if(refresh_str && *refresh_str) {
+        if(!strcmp(refresh_str, "auto")) {
+            if(rc) refresh = rc->update_every;
+            else if(options & RRDR_OPTION_NOT_ALIGNED)
+                refresh = st->update_every;
+            else {
+                refresh = (int)(before - after);
+                if(refresh < 0) refresh = -refresh;
+            }
+        }
+        else {
+            refresh = str2i(refresh_str);
+            if(refresh < 0) refresh = -refresh;
+        }
+    }
+
+    if(!label) {
+        if(alarm) {
+            char *s = (char *)alarm;
+            while(*s) {
+                if(*s == '_') *s = ' ';
+                s++;
+            }
+            label = alarm;
+        }
+        else if(dimensions) {
+            const char *dim = buffer_tostring(dimensions);
+            if(*dim == '|') dim++;
+            label = dim;
+        }
+        else
+            label = st->name;
+    }
+    if(!units) {
+        if(alarm) {
+            if(rc->units)
+                units = rc->units;
+            else
+                units = "";
+        }
+        else if(options & RRDR_OPTION_PERCENTAGE)
+            units = "%";
+        else
+            units = st->units;
+    }
+
+    debug(D_WEB_CLIENT, "%llu: API command 'badge.svg' for chart '%s', alarm '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', options '0x%08x'"
+          , w->id
+          , chart
+          , alarm?alarm:""
+          , (dimensions)?buffer_tostring(dimensions):""
+          , after
+          , before
+          , points
+          , group
+          , options
+    );
+
+    if(rc) {
+        if (refresh > 0) {
+            buffer_sprintf(w->response.header, "Refresh: %d\r\n", refresh);
+            w->response.data->expires = now_realtime_sec() + refresh;
+        }
+        else buffer_no_cacheable(w->response.data);
+
+        if(!value_color) {
+            switch(rc->status) {
+                case RRDCALC_STATUS_CRITICAL:
+                    value_color = "red";
+                    break;
+
+                case RRDCALC_STATUS_WARNING:
+                    value_color = "orange";
+                    break;
+
+                case RRDCALC_STATUS_CLEAR:
+                    value_color = "brightgreen";
+                    break;
+
+                case RRDCALC_STATUS_UNDEFINED:
+                    value_color = "lightgrey";
+                    break;
+
+                case RRDCALC_STATUS_UNINITIALIZED:
+                    value_color = "#000";
+                    break;
+
+                default:
+                    value_color = "grey";
+                    break;
+            }
+        }
+
+        buffer_svg(w->response.data,
+                label,
+                (isnan(rc->value)||isinf(rc->value)) ? rc->value : rc->value * multiply / divide,
+                units,
+                label_color,
+                value_color,
+                precision,
+                scale,
+                options
+        );
+        ret = 200;
+    }
+    else {
+        time_t latest_timestamp = 0;
+        int value_is_null = 1;
+        calculated_number n = NAN;
+        ret = 500;
+
+        // if the collected value is too old, don't calculate its value
+        if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
+            ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
+                                      , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null);
+
+        // if the value cannot be calculated, show empty badge
+        if (ret != 200) {
+            buffer_no_cacheable(w->response.data);
+            value_is_null = 1;
+            n = 0;
+            ret = 200;
+        }
+        else if (refresh > 0) {
+            buffer_sprintf(w->response.header, "Refresh: %d\r\n", refresh);
+            w->response.data->expires = now_realtime_sec() + refresh;
+        }
+        else buffer_no_cacheable(w->response.data);
+
+        // render the badge
+        buffer_svg(w->response.data,
+                label,
+                (value_is_null)?NAN:(n * multiply / divide),
+                units,
+                label_color,
+                value_color,
+                precision,
+                scale,
+                options
+        );
+    }
+
+    cleanup:
+    buffer_free(dimensions);
+    return ret;
+}
diff --git a/web/api/badges/web_buffer_svg.h b/web/api/badges/web_buffer_svg.h
index cd77c0ac3e..f75677bf97 100644
--- a/web/api/badges/web_buffer_svg.h
+++ b/web/api/badges/web_buffer_svg.h
@@ -3,9 +3,14 @@
 #ifndef NETDATA_WEB_BUFFER_SVG_H
 #define NETDATA_WEB_BUFFER_SVG_H 1
 
-#include "web/api/web_api_v1.h"
+#include "libnetdata/libnetdata.h"
+#include "web/server/web_client.h"
 
 extern void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options);
 extern char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision);
 
+extern int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url);
+
+#include "web/api/web_api_v1.h"
+
 #endif /* NETDATA_WEB_BUFFER_SVG_H */
diff --git a/web/api/exporters/Makefile.am b/web/api/exporters/Makefile.am
new file mode 100644
index 0000000000..19554bed8e
--- /dev/null
+++ b/web/api/exporters/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+	README.md \
+	$(NULL)
diff --git a/web/api/exporters/README.md b/web/api/exporters/README.md
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/web/api/exporters/README.md
diff --git a/web/api/exporters/allmetrics.c b/web/api/exporters/allmetrics.c
new file mode 100644
index 0000000000..a426db0cc9
--- /dev/null
+++ b/web/api/exporters/allmetrics.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "allmetrics.h"
+
+struct prometheus_output_options {
+    char *name;
+    PROMETHEUS_OUTPUT_OPTIONS flag;
+} prometheus_output_flags_root[] = {
+        { "help",       PROMETHEUS_OUTPUT_HELP       },
+        { "types",      PROMETHEUS_OUTPUT_TYPES      },
+        { "names",      PROMETHEUS_OUTPUT_NAMES      },
+        { "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },
+        { "variables",  PROMETHEUS_OUTPUT_VARIABLES  },
+
+        // terminator
+        { NULL, PROMETHEUS_OUTPUT_NONE },
+};
+
+inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
+    int format = ALLMETRICS_SHELL;
+    const char *prometheus_server = w->client_ip;
+    uint32_t prometheus_backend_options = global_backend_options;
+    PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = PROMETHEUS_OUTPUT_TIMESTAMPS | ((global_backend_options & BACKEND_OPTION_SEND_NAMES)?PROMETHEUS_OUTPUT_NAMES:0);
+    const char *prometheus_prefix = global_backend_prefix;
+
+    while(url) {
+        char *value = mystrsep(&url, "?&");
+        if (!value || !*value) continue;
+
+        char *name = mystrsep(&value, "=");
+        if(!name || !*name) continue;
+        if(!value || !*value) continue;
+
+        if(!strcmp(name, "format")) {
+            if(!strcmp(value, ALLMETRICS_FORMAT_SHELL))
+                format = ALLMETRICS_SHELL;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS))
+                format = ALLMETRICS_PROMETHEUS;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS))
+                format = ALLMETRICS_PROMETHEUS_ALL_HOSTS;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_JSON))
+                format = ALLMETRICS_JSON;
+            else
+                format = 0;
+        }
+        else if(!strcmp(name, "server")) {
+            prometheus_server = value;
+        }
+        else if(!strcmp(name, "prefix")) {
+            prometheus_prefix = value;
+        }
+        else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) {
+            prometheus_backend_options = backend_parse_data_source(value, prometheus_backend_options);
+        }
+        else {
+            int i;
+            for(i = 0; prometheus_output_flags_root[i].name ; i++) {
+                if(!strcmp(name, prometheus_output_flags_root[i].name)) {
+                    if(!strcmp(value, "yes") || !strcmp(value, "1") || !strcmp(value, "true"))
+                        prometheus_output_options |= prometheus_output_flags_root[i].flag;
+                    else
+                        prometheus_output_options &= ~prometheus_output_flags_root[i].flag;
+
+                    break;
+                }
+            }
+        }
+    }
+
+    buffer_flush(w->response.data);
+    buffer_no_cacheable(w->response.data);
+
+    switch(format) {
+        case ALLMETRICS_JSON:
+            w->response.data->contenttype = CT_APPLICATION_JSON;
+            rrd_stats_api_v1_charts_allmetrics_json(host, w->response.data);
+            return 200;
+
+        case ALLMETRICS_SHELL:
+            w->response.data->contenttype = CT_TEXT_PLAIN;
+            rrd_stats_api_v1_charts_allmetrics_shell(host, w->response.data);
+            return 200;
+
+        case ALLMETRICS_PROMETHEUS:
+            w->response.data->contenttype = CT_PROMETHEUS;
+            rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
+                    host
+                    , w->response.data
+                    , prometheus_server
+                    , prometheus_prefix
+                    , prometheus_backend_options
+                    , prometheus_output_options
+            );
+            return 200;
+
+        case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
+            w->response.data->contenttype = CT_PROMETHEUS;
+            rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
+                    host
+                    , w->response.data
+                    , prometheus_server
+                    , prometheus_prefix
+                    , prometheus_backend_options
+                    , prometheus_output_options
+            );
+            return 200;
+
+        default:
+            w->response.data->contenttype = CT_TEXT_PLAIN;
+            buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "', '" ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported.");
+            return 400;
+    }
+}
diff --git a/web/api/exporters/allmetrics.h b/web/api/exporters/allmetrics.h
new file mode 100644
index 0000000000..bb3494a3fd
--- /dev/null
+++ b/web/api/exporters/allmetrics.h
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_ALLMETRICS_H
+#define NETDATA_API_ALLMETRICS_H
+
+#include "../rrd2json.h"
+#include "shell/allmetrics_shell.h"
+
+extern int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url);
+
+#endif //NETDATA_API_ALLMETRICS_H
diff --git a/web/api/exporters/prometheus/Makefile.am b/web/api/exporters/prometheus/Makefile.am
new file mode 100644
index 0000000000..19554bed8e
--- /dev/null
+++ b/web/api/exporters/prometheus/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+di