diff options
-rw-r--r-- | daemon/global_statistics.c | 145 | ||||
-rw-r--r-- | daemon/global_statistics.h | 21 | ||||
-rw-r--r-- | web/api/queries/README.md | 122 | ||||
-rw-r--r-- | web/api/queries/query.c | 6 | ||||
-rw-r--r-- | web/api/queries/rrdr.h | 3 |
5 files changed, 262 insertions, 35 deletions
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index ff980e43ce..68933e1957 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -2,7 +2,26 @@ #include "common.h" -volatile struct global_statistics global_statistics = { +#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01 + + +static struct global_statistics { + volatile uint16_t connected_clients; + + volatile uint64_t web_requests; + volatile uint64_t web_usec; + volatile uint64_t web_usec_max; + volatile uint64_t bytes_received; + volatile uint64_t bytes_sent; + volatile uint64_t content_size; + volatile uint64_t compressed_content_size; + + volatile uint64_t web_client_count; + + volatile uint64_t rrdr_queries_made; + volatile uint64_t rrdr_db_points_read; + volatile uint64_t rrdr_result_points_generated; +} global_statistics = { .connected_clients = 0, .web_requests = 0, .web_usec = 0, @@ -10,18 +29,45 @@ volatile struct global_statistics global_statistics = { .bytes_sent = 0, .content_size = 0, .compressed_content_size = 0, - .web_client_count = 1 + .web_client_count = 1, + + .rrdr_queries_made = 0, + .rrdr_db_points_read = 0, + .rrdr_result_points_generated = 0, }; +#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS) +#else netdata_mutex_t global_statistics_mutex = NETDATA_MUTEX_INITIALIZER; -inline void global_statistics_lock(void) { +static inline void global_statistics_lock(void) { netdata_mutex_lock(&global_statistics_mutex); } -inline void global_statistics_unlock(void) { +static inline void global_statistics_unlock(void) { netdata_mutex_unlock(&global_statistics_mutex); } +#endif + + +void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated) { +#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS) + __atomic_fetch_add(&global_statistics.rrdr_queries_made, 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&global_statistics.rrdr_db_points_read, db_points_read, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_SEQ_CST); +#else + #warning NOT using atomic operations - using locks for global statistics + if (web_server_is_multithreaded) + global_statistics_lock(); + + global_statistics.rrdr_queries_made++; + global_statistics.rrdr_db_points_read += db_points_read; + global_statistics.rrdr_result_points_generated += result_points_generated; + + if (web_server_is_multithreaded) + global_statistics_unlock(); +#endif +} void finished_web_request_statistics(uint64_t dt, uint64_t bytes_received, @@ -92,17 +138,21 @@ void web_client_disconnected(void) { } -inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) { +static inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) { #if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS) - gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST); - gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST); - gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST); - gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST); - gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST); - gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST); - gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST); - gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST); - gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST); + gs->connected_clients = __atomic_fetch_add(&global_statistics.connected_clients, 0, __ATOMIC_SEQ_CST); + gs->web_requests = __atomic_fetch_add(&global_statistics.web_requests, 0, __ATOMIC_SEQ_CST); + gs->web_usec = __atomic_fetch_add(&global_statistics.web_usec, 0, __ATOMIC_SEQ_CST); + gs->web_usec_max = __atomic_fetch_add(&global_statistics.web_usec_max, 0, __ATOMIC_SEQ_CST); + gs->bytes_received = __atomic_fetch_add(&global_statistics.bytes_received, 0, __ATOMIC_SEQ_CST); + gs->bytes_sent = __atomic_fetch_add(&global_statistics.bytes_sent, 0, __ATOMIC_SEQ_CST); + gs->content_size = __atomic_fetch_add(&global_statistics.content_size, 0, __ATOMIC_SEQ_CST); + gs->compressed_content_size = __atomic_fetch_add(&global_statistics.compressed_content_size, 0, __ATOMIC_SEQ_CST); + gs->web_client_count = __atomic_fetch_add(&global_statistics.web_client_count, 0, __ATOMIC_SEQ_CST); + + gs->rrdr_queries_made = __atomic_fetch_add(&global_statistics.rrdr_queries_made, 0, __ATOMIC_SEQ_CST); + gs->rrdr_db_points_read = __atomic_fetch_add(&global_statistics.rrdr_db_points_read, 0, __ATOMIC_SEQ_CST); + gs->rrdr_result_points_generated = __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, 0, __ATOMIC_SEQ_CST); if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) { uint64_t n = 0; @@ -413,4 +463,71 @@ void global_statistics_charts(void) { rrdset_done(st_compression); } + + // ---------------------------------------------------------------- + + if(gs.rrdr_queries_made) { + static RRDSET *st_rrdr_queries = NULL; + static RRDDIM *rd_queries = NULL; + + if (unlikely(!st_rrdr_queries)) { + st_rrdr_queries = rrdset_create_localhost( + "netdata" + , "queries" + , NULL + , "queries" + , NULL + , "NetData API Queries" + , "queries/s" + , "netdata" + , "stats" + , 130500 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_queries = rrddim_add(st_rrdr_queries, "queries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_rrdr_queries); + + rrddim_set_by_pointer(st_rrdr_queries, rd_queries, (collected_number)gs.rrdr_queries_made); + + rrdset_done(st_rrdr_queries); + } + + // ---------------------------------------------------------------- + + if(gs.rrdr_db_points_read || gs.rrdr_result_points_generated) { + static RRDSET *st_rrdr_points = NULL; + static RRDDIM *rd_points_read = NULL; + static RRDDIM *rd_points_generated = NULL; + + if (unlikely(!st_rrdr_points)) { + st_rrdr_points = rrdset_create_localhost( + "netdata" + , "db_points" + , NULL + , "queries" + , NULL + , "NetData API Points" + , "points/s" + , "netdata" + , "stats" + , 130501 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + + rd_points_read = rrddim_add(st_rrdr_points, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_points_generated = rrddim_add(st_rrdr_points, "generated", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_rrdr_points); + + rrddim_set_by_pointer(st_rrdr_points, rd_points_read, (collected_number)gs.rrdr_db_points_read); + rrddim_set_by_pointer(st_rrdr_points, rd_points_generated, (collected_number)gs.rrdr_result_points_generated); + + rrdset_done(st_rrdr_points); + } } diff --git a/daemon/global_statistics.h b/daemon/global_statistics.h index 753bb5cf96..9dd7db51aa 100644 --- a/daemon/global_statistics.h +++ b/daemon/global_statistics.h @@ -8,24 +8,8 @@ // ---------------------------------------------------------------------------- // global statistics -struct global_statistics { - volatile uint16_t connected_clients; +extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_generated); - volatile uint64_t web_requests; - volatile uint64_t web_usec; - volatile uint64_t web_usec_max; - volatile uint64_t bytes_received; - volatile uint64_t bytes_sent; - volatile uint64_t content_size; - volatile uint64_t compressed_content_size; - - volatile uint64_t web_client_count; -}; - -extern volatile struct global_statistics global_statistics; - -extern void global_statistics_lock(void); -extern void global_statistics_unlock(void); extern void finished_web_request_statistics(uint64_t dt, uint64_t bytes_received, uint64_t bytes_sent, @@ -34,9 +18,6 @@ extern void finished_web_request_statistics(uint64_t dt, extern uint64_t web_client_connected(void); extern void web_client_disconnected(void); - -#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01 -extern void global_statistics_copy(struct global_statistics *gs, uint8_t options); extern void global_statistics_charts(void); #endif /* NETDATA_GLOBAL_STATISTICS_H */ diff --git a/web/api/queries/README.md b/web/api/queries/README.md index e0a5707581..22d2234ed8 100644 --- a/web/api/queries/README.md +++ b/web/api/queries/README.md @@ -1,4 +1,124 @@ # Database Queries -TBD +Netdata database can be queried with `/api/v1/data` and `/api/v1/badge.svg` API methods. + +Every data query accepts the following parameters: + +name|description +:----:|:----: +`chart`|The chart to be queried. +`points`|The number of points to be returned. Netdata can reduce number of points by applying query grouping methods. +`before`|The absolute timestamp or the relative (to now) time the query should finish evaluating data. +`after`|The absolute timestamp or the relative (to `before`) time the query should start evaluating data. +`group`|The grouping method to use when reducing the points the database has. +`gtime`|A resampling period to change the units of the metrics (i.e. setting this to `60` will convert `per second` metrics to `per minute`. +`options`|A bitmap of options that can affect the operation of the query. Only 2 options are used by the query engine: `unaligned` and `percentage`. All the other options are used by the output formatters. +`dimensions`|A simple pattern to filter the dimensions to be queried. + +## Operation + +The query engine works as follows (in this order): + +1. **Identify the exact time-frame required, in absolute timestamps.** + + `after` and `before` define a time-frame: + + - in **absolute timestamps** (unix timestamps, i.e. seconds since epoch). + + - in **relative timestamps**: + + `before` is relative to now and `after` is relative to `before`. + + So, `before=-60&after=-60` evaluates to the time-frame from -120 up to -60 seconds in + the past, relative to now. + + At the end of this operation, `after` and `before` are absolute timestamps. + The engine verifies that the time-frame is available at the database. If it is not, + it will adjust `after` and `before` accordingly so that usable data can be returned, + or no data at all if the time-frame is entirely outside the current range of the + database. + +2. **Identify the grouping of database points required.** + + Grouping database points is used when the caller requests a longer time-frame to be + expressed with fewer points, compared to what is available at the database. + + There are 2 uses of this (that can be combined): + + - The caller requests a specific number of `points` to be returned. + + For example, for a time-frame of 10 minutes, the database has 600 points (1/sec), + while the caller requested these 10 minutes to be expressed in 200 points. + + This feature is used by netdata dashboards when you zoom-out the charts. + The dashboard is requesting the number of points the user's screen has, and netdata + returns that many points to perfectly match the screen. This saves bandwidth + and makes drawing the charts a lot faster. + + - The caller requests a **re-sampling** of the database, by setting `gtime` to any value + above `1`. For example, the database maintains the metrics in the form of `X/sec` + but the caller set `gtime=60` to get `X/min`. + + Using the above information the query engine tries to find a best fit for database-points + to result-points ratio (we call this `group points`). It always tries to keep `group points` + an integer. Keep in mind the query engine may alter a bit `after` if required. So, the engine + may decide to shift the starting point of the time-frame to keep the query optimal. + +3. **Align the time-frame.** + + Alignment is a very important aspect of netdata queries. Without it, the animated + charts on the dashboards would constantly change shape during incremental updates. + To provide consistent grouping of all points, the query engine (by default) aligns + `after` and `before` to be a multiple of `group points`. + + For example, if `group points` is 60 and alignment is enabled, the engine will return + each point with durations XX:XX:00 - XX:XX:59 matching minutes. Of course, depending + on the database granularity for the specific chart and the requested points to be + returned, the engine may use any integer number for `group points`. + + To disable alignment, pass `&options=unaligned` to the query. + +4. **Execute the query** + + To execute the query, the engine evaluates all dimensions of the chart, one after another. + The engine will not evaluate dimensions that do not match the simple pattern given at + the `dimensions` parameter, except when `options=percentage` is given (this option requires + all the dimensions to be evaluated to find the percentage of each dimension vs to chart + total). + + For each dimension, it starts evaluating values from `after` towards `before`. + For each value it calls the **grouping method** specified (the default is `average`). + +## Grouping methods + +The following grouping methods are supported. These are given all the values in the time-frame +and they group the values every `group points`. + +name|identifier(s)|description +:---:|:---:|:---: +Min|`min`|finds the minimum value +Max|`max`|finds the maximum value +Average|`average` `mean`|finds the average value +Sum|`sum`|adds all the values and returns the sum +Median|`median`|sorts the values and returns the value in the middle of the list +Standard Deviation|`stddev`|finds the standard deviation of the values +Coefficient of Variation|`cv` `rds`|finds the relative standard deviation of the values +Single Exponential Smoothing|`ses` `ema` `ewma`|finds the exponential weighted moving average of the values +Double Exponential Smoothing|`des`|applies Holt-Winters double exponential smoothing +Incremental Sum|`incremental_sum` `incremental-sum`|find the difference of the last vs the first values + +## Further processing + +The result of the query engine is always a structure that has dimensions and values +for each dimension. + +Formatting modules are then used to convert this result in many different formats and return it +to the caller. + +## Performance + +The query engine is highly optimized for speed. Most of its modules implement "online" +versions of the algorithms, requiring just one pass on the database values to produce +the result. + diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 764375932a..1806b6d4e9 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -408,6 +408,7 @@ static inline void do_dimension( RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING; + size_t db_points_read = 0; for( ; points_added < points_wanted ; now += dt, slot++ ) { if(unlikely(slot >= entries)) slot = 0; @@ -442,6 +443,7 @@ static inline void do_dimension( // add this value for grouping r->internal.grouping_add(r, value); values_in_group++; + db_points_read++; if(unlikely(values_in_group == group_size)) { rrdr_line = rrdr_line_init(r, now, rrdr_line); @@ -469,6 +471,9 @@ static inline void do_dimension( } } + r->internal.db_points_read += db_points_read; + r->internal.result_points_generated += points_added; + r->before = max_date; r->after = min_date; rrdr_done(r, rrdr_line); @@ -943,5 +948,6 @@ RRDR *rrd2rrdr( } } + rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); return r; } diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 44e14fe19f..a0295db463 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -85,6 +85,9 @@ typedef struct rrdresult { #ifdef NETDATA_INTERNAL_CHECKS const char *log; #endif + + size_t db_points_read; + size_t result_points_generated; } internal; } RRDR; |