Add a timeout parameter to data queries (#12649)

* Add timeout parameter in queries and in calling functions

* Add CANCEL flag in RRDR and code to cancel a query

* Update swagger

* Format swagger file properly

8 files changed, 74 insertions, 12 deletions

diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index f8c0a17a65..65ca21d196 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -1102,7 +1102,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
         // if the collected value is too old, don't calculate its value
         if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above)))
             ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL
-                                      , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null);
+                                      , points, after, before, group, 0, options, NULL, &latest_timestamp, &value_is_null, 0);
 
         // if the value cannot be calculated, show empty badge
         if (ret != HTTP_RESP_OK) {
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index ad0001c73e..1a8b07c7c8 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -167,9 +167,10 @@ int rrdset2value_api_v1(
         , time_t *db_after
         , time_t *db_before
         , int *value_is_null
+        , int timeout
 ) {
 
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL);
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
 
     if(!r) {
         if(value_is_null) *value_is_null = 1;
@@ -218,17 +219,23 @@ int rrdset2anything_api_v1(
         , struct context_param *context_param_list
         , char *chart_label_key
         , int max_anomaly_rates
-) {
-
+        , int timeout
+)
+{
     if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
         st->last_accessed_time = now_realtime_sec();
 
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
     }
 
+    if (r->result_options & RRDR_RESULT_OPTION_CANCEL) {
+        rrdr_free(r);
+        return HTTP_RESP_BACKEND_FETCH_FAILED;
+    }
+
     if (st && st->state && st->state->is_ar_chart)
         ml_process_rrdr(r, max_anomaly_rates);
 
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index c24547ca63..af809c54f9 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -68,6 +68,7 @@ extern int rrdset2anything_api_v1(
         , struct context_param *context_param_list
         , char *chart_label_key
         , int max_anomaly_rates
+        , int timeout
 );
 
 extern int rrdset2value_api_v1(
@@ -84,6 +85,7 @@ extern int rrdset2value_api_v1(
         , time_t *db_after
         , time_t *db_before
         , int *value_is_null
+        , int timeout
 );
 
 extern void build_context_param_list(struct context_param **param_list, RRDSET *st);
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 0569e515a0..64c3a95bcf 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -232,6 +232,18 @@
             }
           },
           {
+            "name": "timeout",
+            "in": "query",
+            "description": "Specify a timeout value in milliseconds after which the agent will abort the query and return a 503 error. A value of 0 indicates no timeout.",
+            "required": false,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 0
+            }
+          },
+          {
             "name": "format",
             "in": "query",
             "description": "The format of the data to be returned.",
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index 7d052c26f9..a03180ea08 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -202,6 +202,16 @@ paths:
             type: number
             format: integer
             default: 0
+        - name: timeout
+          in: query
+          description: Specify a timeout value in milliseconds after which the agent will
+            abort the query and return a 503 error. A value of 0 indicates no timeout.
+          required: false
+          allowEmptyValue: false
+          schema:
+            type: number
+            format: integer
+            default: 0
         - name: format
           in: query
           description: The format of the data to be returned.
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 8272e083f4..c55a970609 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -844,6 +844,7 @@ static RRDR *rrd2rrdr_fixedstep(
         , time_t last_entry_t
         , int absolute_period_requested
         , struct context_param *context_param_list
+        , int timeout
 ) {
     int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
 
@@ -1097,6 +1098,10 @@ static RRDR *rrd2rrdr_fixedstep(
 
     RRDDIM *rd;
     long c, dimensions_used = 0, dimensions_nonzero = 0;
+    struct timeval query_start_time;
+    struct timeval query_current_time;
+    if (timeout)
+        now_realtime_timeval(&query_start_time);
     for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
 
         // if we need a percentage, we need to calculate all dimensions
@@ -1118,6 +1123,8 @@ static RRDR *rrd2rrdr_fixedstep(
                 , before_wanted
                 , options
                 );
+        if (timeout)
+            now_realtime_timeval(&query_current_time);
 
         if(r->od[c] & RRDR_DIMENSION_NONZERO)
             dimensions_nonzero++;
@@ -1155,6 +1162,12 @@ static RRDR *rrd2rrdr_fixedstep(
         }
 
         dimensions_used++;
+        if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
+            log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
+                       dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
+            r->result_options |= RRDR_RESULT_OPTION_CANCEL;
+            break;
+        }
     }
 
     #ifdef NETDATA_INTERNAL_CHECKS
@@ -1188,7 +1201,7 @@ static RRDR *rrd2rrdr_fixedstep(
     r->internal.grouping_free(r);
 
     // when all the dimensions are zero, we should return all of them
-    if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
+    if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
         // all the dimensions are zero
         // mark them as NONZERO to send them all
         for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
@@ -1217,6 +1230,7 @@ static RRDR *rrd2rrdr_variablestep(
         , int absolute_period_requested
         , struct rrdeng_region_info *region_info_array
         , struct context_param *context_param_list
+        , int timeout
 ) {
     int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
 
@@ -1474,6 +1488,10 @@ static RRDR *rrd2rrdr_variablestep(
 
     RRDDIM *rd;
     long c, dimensions_used = 0, dimensions_nonzero = 0;
+    struct timeval query_start_time;
+    struct timeval query_current_time;
+    if (timeout)
+        now_realtime_timeval(&query_start_time);
     for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
 
         // if we need a percentage, we need to calculate all dimensions
@@ -1495,6 +1513,8 @@ static RRDR *rrd2rrdr_variablestep(
                 , before_wanted
                 , options
         );
+        if (timeout)
+            now_realtime_timeval(&query_current_time);
 
         if(r->od[c] & RRDR_DIMENSION_NONZERO)
             dimensions_nonzero++;
@@ -1532,6 +1552,12 @@ static RRDR *rrd2rrdr_variablestep(
         }
 
         dimensions_used++;
+        if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
+            log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
+                       dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
+            r->result_options |= RRDR_RESULT_OPTION_CANCEL;
+            break;
+        }
     }
 
     #ifdef NETDATA_INTERNAL_CHECKS
@@ -1566,7 +1592,7 @@ static RRDR *rrd2rrdr_variablestep(
     r->internal.grouping_free(r);
 
     // when all the dimensions are zero, we should return all of them
-    if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
+    if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
         // all the dimensions are zero
         // mark them as NONZERO to send them all
         for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
@@ -1591,6 +1617,7 @@ RRDR *rrd2rrdr(
         , RRDR_OPTIONS options
         , const char *dimensions
         , struct context_param *context_param_list
+        , int timeout
 )
 {
     int rrd_update_every;
@@ -1644,7 +1671,7 @@ RRDR *rrd2rrdr(
             }
             return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
                                       resampling_time_requested, options, dimensions, rrd_update_every,
-                                      first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
+                                      first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
         } else {
             if (rrd_update_every != (uint16_t)max_interval) {
                 rrd_update_every = (uint16_t) max_interval;
@@ -1655,11 +1682,11 @@ RRDR *rrd2rrdr(
             }
             return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
                                          resampling_time_requested, options, dimensions, rrd_update_every,
-                                         first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list);
+                                         first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout);
         }
     }
 #endif
     return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
                               resampling_time_requested, options, dimensions,
-                              rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
+                              rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout);
 }
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index 3637df6878..bd94e56e2b 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -47,6 +47,7 @@ typedef enum rrdr_result_flags {
     RRDR_RESULT_OPTION_RELATIVE      = 0x00000002, // the query uses relative time-frames
                                                    // (should not to be cached by browsers and proxies)
     RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
+    RRDR_RESULT_OPTION_CANCEL        = 0x00000008, // the query needs to be cancelled
 } RRDR_RESULT_FLAGS;
 
 typedef struct rrdresult {
@@ -110,7 +111,7 @@ extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *contex
 extern RRDR *rrd2rrdr(
     RRDSET *st, long points_requested, long long after_requested, long long before_requested,
     RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
-    struct context_param *context_param_list);
+    struct context_param *context_param_list, int timeout);
 
 #include "query.h"
 
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index 26700d8fec..20318a1318 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -415,6 +415,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     char *after_str = NULL;
     char *group_time_str = NULL;
     char *points_str = NULL;
+    char *timeout_str = NULL;
     char *max_anomaly_rates_str = NULL;
     char *context = NULL;
     char *chart_label_key = NULL;
@@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         else if(!strcmp(name, "after")) after_str = value;
         else if(!strcmp(name, "before")) before_str = value;
         else if(!strcmp(name, "points")) points_str = value;
+        else if(!strcmp(name, "timeout")) timeout_str = value;
         else if(!strcmp(name, "gtime")) group_time_str = value;
         else if(!strcmp(name, "group")) {
             group = web_client_api_request_v1_data_group(value, RRDR_GROUPING_AVERAGE);
@@ -576,6 +578,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     long long before = (before_str && *before_str)?str2l(before_str):0;
     long long after  = (after_str  && *after_str) ?str2l(after_str):-600;
     int       points = (points_str && *points_str)?str2i(points_str):0;
+    int       timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;
     long      group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0;
     int       max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0;
 
@@ -623,7 +626,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format,
                                  points, after, before, group, group_time,
                                  options, &last_timestamp_in_data, context_param_list,
-                                 chart_label_key, max_anomaly_rates);
+                                 chart_label_key, max_anomaly_rates, timeout);
 
     free_context_param_list(&context_param_list);