/api/v2/X improvements part 3 (#14665)

* max web request size to 64KB

* fix the request too big message

* increase max request reading tries to 100

* support for bigger web requests

* add "avg" as a shortcut for "average" to both group by aggregation and time aggregation; discard the last partial points of a query in play mode, up to max update every; group by hidden dimensions too

* better implementation for partial data trimming

* added group_by=selected to return only one dimension for all selected metrics

* fix acceptance of group_by=selected

* passing option "raw" disables partial data trimming

* remove obsolete option "plan"; use "debug"

* fix view.min and view.max calculation - there were 2 bugs: a) min and max were reset for every row and b) min and max were corrupted by GBC and AR printing

* per row annotations

* added time column to point annotations

* disable caching for /api/v2/contexts responses

* added api format json2 that returns an array for each points, having all the point values and annotations in them

* work on swagger about /api/v2

* prevent infinite loop

* cleanup and swagger work

* allow negative simple pattern expressions to work as expected

* do not lookup in the dictionary empty names

* garbage collect dictionaries

* make query_target allocate less aggressively; queries fill the remaining points with nulls

* reusable query ops to save memory on huge queries

* move parts of query plans into query ops to save query target memory

* remove storage engine from query metric tiers, to save memory, and recalculate it when it is needed

1 files changed, 5 insertions, 11 deletions

diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h
index ff9c1c8828..0261c78227 100644
--- a/database/contexts/rrdcontext.h
+++ b/database/contexts/rrdcontext.h
@@ -149,17 +149,9 @@ typedef struct query_plan_entry {
     size_t tier;
     time_t after;
     time_t before;
-    time_t expanded_after;
-    time_t expanded_before;
-    struct storage_engine_query_handle handle;
-    STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
-    int (*is_finished)(struct storage_engine_query_handle *handle);
-    void (*finalize)(struct storage_engine_query_handle *handle);
-    bool initialized;
-    bool finalized;
 } QUERY_PLAN_ENTRY;
 
-#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS * 2)
+#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS)
 
 struct query_metrics_counts {
     size_t selected;
@@ -234,7 +226,6 @@ typedef struct query_metric {
     RRDR_DIMENSION_FLAGS status;
 
     struct query_metric_tier {
-        struct storage_engine *eng;
         STORAGE_METRIC_HANDLE *db_metric_handle;
         time_t db_first_time_s;         // the oldest timestamp available for this tier
         time_t db_last_time_s;          // the latest timestamp available for this tier
@@ -248,7 +239,7 @@ typedef struct query_metric {
     } plan;
 
     struct {
-        uint32_t query_host_id;
+        uint32_t query_node_id;
         uint32_t query_context_id;
         uint32_t query_instance_id;
         uint32_t query_dimension_id;
@@ -334,6 +325,7 @@ typedef struct query_target {
     size_t queries;                         // how many query we have done so far with this QUERY_TARGET - not related to database queries
 
     struct {
+        time_t now;                         // the current timestamp, the absolute max for any query timestamp
         bool relative;                      // true when the request made with relative timestamps, true if it was absolute
         bool aligned;
         time_t after;                       // the absolute timestamp this query is about
@@ -452,6 +444,8 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
     return rrdmetric_acquired_name(qd->rma);
 }
 
+struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
+
 STRING *query_instance_id_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);
 STRING *query_instance_name_fqdn(QUERY_TARGET *qt, QUERY_INSTANCE *qi);