diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-04-07 21:25:01 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-04-07 21:25:01 +0300 |
commit | 204dd9ae272445d13f308badb07e99675fa34892 (patch) | |
tree | f42e873c60219b5031dcfc3e076adb2398cdb3fe /database/contexts/rrdcontext.h | |
parent | 61925baaf6e2448c641e8e71248a47f7a01c4efd (diff) |
Boost dbengine (#14832)
* configure extent cache size
* workers can now execute up to 10 jobs in a run, boosting query prep and extent reads
* fix dispatched and executing counters
* boost to the max
* increase libuv worker threads
* query prep always get more prio than extent reads; stop processing in batch when dbengine is queue is critical
* fix accounting of query prep
* inlining of time-grouping functions, to speed up queries with billions of points
* make switching based on a local const variable
* print one pending contexts loading message per iteration
* inlined store engine query API
* inlined storage engine data collection api
* inlined all storage engine query ops
* eliminate and inline data collection ops
* simplified query group-by
* more error handling
* optimized partial trimming of group-by queries
* preparative work to support multiple passes of group-by
* more preparative work to support multiple passes of group-by (accepts multiple group-by params)
* unified query timings
* unified query timings - weights endpoint
* query target is no longer a static thread variable - there is a list of cached query targets, each of which of freed every 1000 queries
* fix query memory accounting
* added summary.dimension[].pri and sorted summary.dimensions based on priority and then name
* limit max ACLK WEB response size to 30MB
* the response type should be text/plain
* more preparative work for multiple group-by passes
* create functions for generating group by keys, ids and names
* multiple group-by passes are now supported
* parse group-by options array also with an index
* implemented percentage-of-instance group by function
* family is now merged in multi-node contexts
* prevent uninitialized use
Diffstat (limited to 'database/contexts/rrdcontext.h')
-rw-r--r-- | database/contexts/rrdcontext.h | 77 |
1 files changed, 54 insertions, 23 deletions
diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h index f42c43135b..5093e1d614 100644 --- a/database/contexts/rrdcontext.h +++ b/database/contexts/rrdcontext.h @@ -201,6 +201,7 @@ typedef struct query_instance { typedef struct query_dimension { uint32_t slot; + uint32_t priority; RRDMETRIC_ACQUIRED *rma; QUERY_STATUS status; } QUERY_DIMENSION; @@ -231,7 +232,8 @@ typedef struct query_metric { STORAGE_POINT query_points; struct { - size_t slot; + uint32_t slot; + uint32_t first_slot; STRING *id; STRING *name; STRING *units; @@ -241,9 +243,16 @@ typedef struct query_metric { } QUERY_METRIC; #define MAX_QUERY_TARGET_ID_LENGTH 255 +#define MAX_QUERY_GROUP_BY_PASSES 2 typedef bool (*qt_interrupt_callback_t)(void *data); +struct group_by_pass { + RRDR_GROUP_BY group_by; + char *group_by_label; + RRDR_GROUP_BY_FUNCTION aggregation; +}; + typedef struct query_target_request { size_t version; @@ -284,9 +293,7 @@ typedef struct query_target_request { const char *time_group_options; // group by across multiple time-series - RRDR_GROUP_BY group_by; - char *group_by_label; - RRDR_GROUP_BY_FUNCTION group_by_aggregate_function; + struct group_by_pass group_by[MAX_QUERY_GROUP_BY_PASSES]; usec_t received_ut; @@ -313,15 +320,19 @@ struct query_versions { uint64_t alerts_soft_hash; }; +struct query_timings { + usec_t received_ut; + usec_t preprocessed_ut; + usec_t executed_ut; + usec_t finished_ut; +}; + #define query_view_update_every(qt) ((qt)->window.group * (qt)->window.query_granularity) typedef struct query_target { char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging) QUERY_TARGET_REQUEST request; - bool used; // when true, this query is currently being used - size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries - struct { time_t now; // the current timestamp, the absolute max for any query timestamp bool relative; // true when the request made with relative timestamps, true if it was absolute @@ -388,19 +399,20 @@ typedef struct query_target { struct { size_t used; - char *label_keys[GROUP_BY_MAX_LABEL_KEYS]; - } group_by; + char *label_keys[GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES]; + } group_by[MAX_QUERY_GROUP_BY_PASSES]; STORAGE_POINT query_points; - struct query_versions versions; + struct query_timings timings; struct { - usec_t received_ut; - usec_t preprocessed_ut; - usec_t executed_ut; - usec_t finished_ut; - } timings; + SPINLOCK spinlock; + bool used; // when true, this query is currently being used + size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries + struct query_target *prev; + struct query_target *next; + } internal; } QUERY_TARGET; static inline NEVERNULL QUERY_NODE *query_node(QUERY_TARGET *qt, size_t id) { @@ -455,13 +467,6 @@ struct api_v2_contexts_request { char *contexts; char *q; - struct { - usec_t received_ut; - usec_t processing_ut; - usec_t output_ut; - usec_t finished_ut; - } timings; - time_t timeout_ms; qt_interrupt_callback_t interrupt_callback; @@ -479,8 +484,10 @@ typedef enum __attribute__ ((__packed__)) { int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options); RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o); -void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s); +void buffer_json_agents_array_v2(BUFFER *wb, struct query_timings *timings, time_t now_s); void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut); +void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings); +void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings); // ---------------------------------------------------------------------------- // scope @@ -515,5 +522,29 @@ bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t b (((first_entry_s) - ((update_every_s) * 2) <= (before)) && \ ((last_entry_s) + ((update_every_s) * 2) >= (after))) +#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW) + +static inline bool query_target_has_percentage_of_instance(QUERY_TARGET *qt) { + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) + if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + return true; + + return false; +} + +static inline bool query_target_needs_all_dimensions(QUERY_TARGET *qt) { + if(qt->request.options & RRDR_OPTION_PERCENTAGE) + return true; + + return query_target_has_percentage_of_instance(qt); +} + +static inline bool query_target_has_percentage_units(QUERY_TARGET *qt) { + if(qt->window.time_group_method == RRDR_GROUPING_CV || query_target_needs_all_dimensions(qt)) + return true; + + return false; +} + #endif // NETDATA_RRDCONTEXT_H |