summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--daemon/global_statistics.c145
-rw-r--r--daemon/global_statistics.h21
-rw-r--r--web/api/queries/README.md122
-rw-r--r--web/api/queries/query.c6
-rw-r--r--web/api/queries/rrdr.h3
5 files changed, 262 insertions, 35 deletions
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index ff980e43ce..68933e1957 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -2,7 +2,26 @@
#include "common.h"
-volatile struct global_statistics global_statistics = {
+#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
+
+
+static struct global_statistics {
+ volatile uint16_t connected_clients;
+
+ volatile uint64_t web_requests;
+ volatile uint64_t web_usec;
+ volatile uint64_t web_usec_max;
+ volatile uint64_t bytes_received;
+ volatile uint64_t bytes_sent;
+ volatile uint64_t content_size;
+ volatile uint64_t compressed_content_size;
+
+ volatile uint64_t web_client_count;
+
+ volatile uint64_t rrdr_queries_made;
+ volatile uint64_t rrdr_db_points_read;
+ volatile uint64_t rrdr_result_points_generated;
+} global_statistics = {
.connected_clients = 0,
.web_requests = 0,
.web_usec = 0,
@@ -10,18 +29,45 @@ volatile struct global_statistics global_statistics = {
.bytes_sent = 0,
.content_size = 0,
.compressed_content_size = 0,
- .web_client_count = 1
+ .web_client_count = 1,
+
+ .rrdr_queries_made = 0,
+ .rrdr_db_points_read = 0,
+ .rrdr_result_points_generated = 0,
};
+#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
+#else
netdata_mutex_t global_statistics_mutex = NETDATA_MUTEX_INITIALIZER;
-inline void global_statistics_lock(void) {
+static inline void global_statistics_lock(void) {
netdata_mutex_lock(&global_statistics_mutex);
}
-inline void global_statistics_unlock(void) {
+static inline void global_statistics_unlock(void) {
netdata_mutex_unlock(&global_statistics_mutex);
}
+#endif
+
+
+void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated) {
+#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
+ __atomic_fetch_add(&global_statistics.rrdr_queries_made, 1, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&global_statistics.rrdr_db_points_read, db_points_read, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_SEQ_CST);
+#else
+ #warning NOT using atomic operations - using locks for global statistics
+ if (web_server_is_multithreaded)
+ global_statistics_lock();
+
+ global_statistics.rrdr_queries_made++;
+ global_statistics.rrdr_db_points_read += db_points_read;
+ global_statistics.rrdr_result_points_generated += result_points_generated;
+
+ if (web_server_is_multithreaded)
+ global_statistics_unlock();
+#endif
+}
void finished_web_request_statistics(uint64_t dt,
uint64_t bytes_received,
@@ -92,17 +138,21 @@ void web_client_disconnected(void) {
}
-inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
+static inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
- gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST);
- gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST);
- gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST);
- gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST);
- gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST);
- gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST);
- gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST);
- gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST);
- gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST);
+ gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST);
+ gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST);
+ gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST);
+ gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST);
+ gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST);
+ gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST);
+ gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST);
+ gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST);
+ gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST);
+
+ gs->rrdr_queries_made = __atomic_fetch_add(&global_statistics.rrdr_queries_made, 0, __ATOMIC_SEQ_CST);
+ gs->rrdr_db_points_read = __atomic_fetch_add(&global_statistics.rrdr_db_points_read, 0, __ATOMIC_SEQ_CST);
+ gs->rrdr_result_points_generated = __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, 0, __ATOMIC_SEQ_CST);
if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) {
uint64_t n = 0;
@@ -413,4 +463,71 @@ void global_statistics_charts(void) {
rrdset_done(st_compression);
}
+
+ // ----------------------------------------------------------------
+
+ if(gs.rrdr_queries_made) {
+ static RRDSET *st_rrdr_queries = NULL;
+ static RRDDIM *rd_queries = NULL;
+
+ if (unlikely(!st_rrdr_queries)) {
+ st_rrdr_queries = rrdset_create_localhost(
+ "netdata"
+ , "queries"
+ , NULL
+ , "queries"
+ , NULL
+ , "NetData API Queries"
+ , "queries/s"
+ , "netdata"
+ , "stats"
+ , 130500
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_queries = rrddim_add(st_rrdr_queries, "queries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_rrdr_queries);
+
+ rrddim_set_by_pointer(st_rrdr_queries, rd_queries, (collected_number)gs.rrdr_queries_made);
+
+ rrdset_done(st_rrdr_queries);
+ }
+
+ // ----------------------------------------------------------------
+
+ if(gs.rrdr_db_points_read || gs.rrdr_result_points_generated) {
+ static RRDSET *st_rrdr_points = NULL;
+ static RRDDIM *rd_points_read = NULL;
+ static RRDDIM *rd_points_generated = NULL;
+
+ if (unlikely(!st_rrdr_points)) {
+ st_rrdr_points = rrdset_create_localhost(
+ "netdata"
+ , "db_points"
+ , NULL
+ , "queries"
+ , NULL
+ , "NetData API Points"
+ , "points/s"
+ , "netdata"
+ , "stats"
+ , 130501
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_points_read = rrddim_add(st_rrdr_points, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_points_generated = rrddim_add(st_rrdr_points, "generated", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_rrdr_points);
+
+ rrddim_set_by_pointer(st_rrdr_points, rd_points_read, (collected_number)gs.rrdr_db_points_read);
+ rrddim_set_by_pointer(st_rrdr_points, rd_points_generated, (collected_number)gs.rrdr_result_points_generated);
+
+ rrdset_done(st_rrdr_points);
+ }
}
diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h
index 753bb5cf96..9dd7db51aa 100644
--- a/daemon/global_statistics.h
+++ b/daemon/global_statistics.h
@@ -8,24 +8,8 @@
// ----------------------------------------------------------------------------
// global statistics
-struct global_statistics {
- volatile uint16_t connected_clients;
+extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated);
- volatile uint64_t web_requests;
- volatile uint64_t web_usec;
- volatile uint64_t web_usec_max;
- volatile uint64_t bytes_received;
- volatile uint64_t bytes_sent;
- volatile uint64_t content_size;
- volatile uint64_t compressed_content_size;
-
- volatile uint64_t web_client_count;
-};
-
-extern volatile struct global_statistics global_statistics;
-
-extern void global_statistics_lock(void);
-extern void global_statistics_unlock(void);
extern void finished_web_request_statistics(uint64_t dt,
uint64_t bytes_received,
uint64_t bytes_sent,
@@ -34,9 +18,6 @@ extern void finished_web_request_statistics(uint64_t dt,
extern uint64_t web_client_connected(void);
extern void web_client_disconnected(void);
-
-#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
-extern void global_statistics_copy(struct global_statistics *gs, uint8_t options);
extern void global_statistics_charts(void);
#endif /* NETDATA_GLOBAL_STATISTICS_H */
diff --git a/web/api/queries/README.md b/web/api/queries/README.md
index e0a5707581..22d2234ed8 100644
--- a/web/api/queries/README.md
+++ b/web/api/queries/README.md
@@ -1,4 +1,124 @@
# Database Queries
-TBD
+Netdata database can be queried with `/api/v1/data` and `/api/v1/badge.svg` API methods.
+
+Every data query accepts the following parameters:
+
+name|description
+:----:|:----:
+`chart`|The chart to be queried.
+`points`|The number of points to be returned. Netdata can reduce number of points by applying query grouping methods.
+`before`|The absolute timestamp or the relative (to now) time the query should finish evaluating data.
+`after`|The absolute timestamp or the relative (to `before`) time the query should start evaluating data.
+`group`|The grouping method to use when reducing the points the database has.
+`gtime`|A resampling period to change the units of the metrics (i.e. setting this to `60` will convert `per second` metrics to `per minute`.
+`options`|A bitmap of options that can affect the operation of the query. Only 2 options are used by the query engine: `unaligned` and `percentage`. All the other options are used by the output formatters.
+`dimensions`|A simple pattern to filter the dimensions to be queried.
+
+## Operation
+
+The query engine works as follows (in this order):
+
+1. **Identify the exact time-frame required, in absolute timestamps.**
+
+ `after` and `before` define a time-frame:
+
+ - in **absolute timestamps** (unix timestamps, i.e. seconds since epoch).
+
+ - in **relative timestamps**:
+
+ `before` is relative to now and `after` is relative to `before`.
+
+ So, `before=-60&after=-60` evaluates to the time-frame from -120 up to -60 seconds in
+ the past, relative to now.
+
+ At the end of this operation, `after` and `before` are absolute timestamps.
+ The engine verifies that the time-frame is available at the database. If it is not,
+ it will adjust `after` and `before` accordingly so that usable data can be returned,
+ or no data at all if the time-frame is entirely outside the current range of the
+ database.
+
+2. **Identify the grouping of database points required.**
+
+ Grouping database points is used when the caller requests a longer time-frame to be
+ expressed with fewer points, compared to what is available at the database.
+
+ There are 2 uses of this (that can be combined):
+
+ - The caller requests a specific number of `points` to be returned.
+
+ For example, for a time-frame of 10 minutes, the database has 600 points (1/sec),
+ while the caller requested these 10 minutes to be expressed in 200 points.
+
+ This feature is used by netdata dashboards when you zoom-out the charts.
+ The dashboard is requesting the number of points the user's screen has, and netdata
+ returns that many points to perfectly match the screen. This saves bandwidth
+ and makes drawing the charts a lot faster.
+
+ - The caller requests a **re-sampling** of the database, by setting `gtime` to any value
+ above `1`. For example, the database maintains the metrics in the form of `X/sec`
+ but the caller set `gtime=60` to get `X/min`.
+
+ Using the above information the query engine tries to find a best fit for database-points
+ to result-points ratio (we call this `group points`). It always tries to keep `group points`
+ an integer. Keep in mind the query engine may alter a bit `after` if required. So, the engine
+ may decide to shift the starting point of the time-frame to keep the query optimal.
+
+3. **Align the time-frame.**
+
+ Alignment is a very important aspect of netdata queries. Without it, the animated
+ charts on the dashboards would constantly change shape during incremental updates.
+ To provide consistent grouping of all points, the query engine (by default) aligns
+ `after` and `before` to be a multiple of `group points`.
+
+ For example, if `group points` is 60 and alignment is enabled, the engine will return
+ each point with durations XX:XX:00 - XX:XX:59 matching minutes. Of course, depending
+ on the database granularity for the specific chart and the requested points to be
+ returned, the engine may use any integer number for `group points`.
+
+ To disable alignment, pass `&options=unaligned` to the query.
+
+4. **Execute the query**
+
+ To execute the query, the engine evaluates all dimensions of the chart, one after another.
+ The engine will not evaluate dimensions that do not match the simple pattern given at
+ the `dimensions` parameter, except when `options=percentage` is given (this option requires
+ all the dimensions to be evaluated to find the percentage of each dimension vs to chart
+ total).
+
+ For each dimension, it starts evaluating values from `after` towards `before`.
+ For each value it calls the **grouping method** specified (the default is `average`).
+
+## Grouping methods
+
+The following grouping methods are supported. These are given all the values in the time-frame
+and they group the values every `group points`.
+
+name|identifier(s)|description
+:---:|:---:|:---:
+Min|`min`|finds the minimum value
+Max|`max`|finds the maximum value
+Average|`average` `mean`|finds the average value
+Sum|`sum`|adds all the values and returns the sum
+Median|`median`|sorts the values and returns the value in the middle of the list
+Standard Deviation|`stddev`|finds the standard deviation of the values
+Coefficient of Variation|`cv` `rds`|finds the relative standard deviation of the values
+Single Exponential Smoothing|`ses` `ema` `ewma`|finds the exponential weighted moving average of the values
+Double Exponential Smoothing|`des`|applies Holt-Winters double exponential smoothing
+Incremental Sum|`incremental_sum` `incremental-sum`|find the difference of the last vs the first values
+
+## Further processing
+
+The result of the query engine is always a structure that has dimensions and values
+for each dimension.
+
+Formatting modules are then used to convert this result in many different formats and return it
+to the caller.
+
+## Performance
+
+The query engine is highly optimized for speed. Most of its modules implement "online"
+versions of the algorithms, requiring just one pass on the database values to produce
+the result.
+
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 764375932a..1806b6d4e9 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -408,6 +408,7 @@ static inline void do_dimension(
RRDR_VALUE_FLAGS
group_value_flags = RRDR_VALUE_NOTHING;
+ size_t db_points_read = 0;
for( ; points_added < points_wanted ; now += dt, slot++ ) {
if(unlikely(slot >= entries)) slot = 0;
@@ -442,6 +443,7 @@ static inline void do_dimension(
// add this value for grouping
r->internal.grouping_add(r, value);
values_in_group++;
+ db_points_read++;
if(unlikely(values_in_group == group_size)) {
rrdr_line = rrdr_line_init(r, now, rrdr_line);
@@ -469,6 +471,9 @@ static inline void do_dimension(
}
}
+ r->internal.db_points_read += db_points_read;
+ r->internal.result_points_generated += points_added;
+
r->before = max_date;
r->after = min_date;
rrdr_done(r, rrdr_line);
@@ -943,5 +948,6 @@ RRDR *rrd2rrdr(
}
}
+ rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
return r;
}
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index 44e14fe19f..a0295db463 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -85,6 +85,9 @@ typedef struct rrdresult {
#ifdef NETDATA_INTERNAL_CHECKS
const char *log;
#endif
+
+ size_t db_points_read;
+ size_t result_points_generated;
} internal;
} RRDR;