diff options
author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2021-03-22 09:47:22 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-03-22 09:47:22 +0200 |
commit | 65bc43d9cbc17d6ac356672cdc24ffe74fc463e5 (patch) | |
tree | 3c6b378578d9dfd911555cdda706529fb2f1b525 | |
parent | 0b0748ee4f9d573e4b741f37a0fa2d158c37599e (diff) |
Add data query support for archived charts (#10771)
-rwxr-xr-x | database/engine/rrdengineapi.c | 30 | ||||
-rw-r--r-- | database/engine/rrdengineapi.h | 1 | ||||
-rw-r--r-- | database/rrd.h | 12 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.c | 225 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.h | 1 | ||||
-rw-r--r-- | web/api/formatters/json/json.c | 10 | ||||
-rw-r--r-- | web/api/formatters/json/json.h | 2 | ||||
-rw-r--r-- | web/api/formatters/json_wrapper.c | 28 | ||||
-rw-r--r-- | web/api/formatters/json_wrapper.h | 2 | ||||
-rw-r--r-- | web/api/formatters/rrd2json.c | 37 | ||||
-rw-r--r-- | web/api/queries/query.c | 27 | ||||
-rw-r--r-- | web/api/queries/rrdr.c | 3 | ||||
-rw-r--r-- | web/api/web_api_v1.c | 13 | ||||
-rw-r--r-- | web/server/web_client.c | 25 |
14 files changed, 345 insertions, 71 deletions
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 7b2ff5b721..cb46e06e3b 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -691,6 +691,36 @@ time_t rrdeng_metric_oldest_time(RRDDIM *rd) return page_index->oldest_time / USEC_PER_SEC; } +int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t) +{ + struct page_cache *pg_cache; + struct rrdengine_instance *ctx; + Pvoid_t *PValue; + struct pg_cache_page_index *page_index = NULL; + + ctx = get_rrdeng_ctx_from_host(localhost); + if (unlikely(!ctx)) { + error("Failed to fetch multidb context"); + return 1; + } + pg_cache = &ctx->pg_cache; + + uv_rwlock_rdlock(&pg_cache->metrics_index.lock); + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t)); + if (likely(NULL != PValue)) { + page_index = *PValue; + } + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock); + + if (likely(page_index)) { + *first_entry_t = page_index->oldest_time / USEC_PER_SEC; + *last_entry_t = page_index->latest_time / USEC_PER_SEC; + return 0; + } + + return 1; +} + /* Also gets a reference for the page */ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrdeng_page_descr **ret_descr) { diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h index 41375b980c..00e55e6625 100644 --- a/database/engine/rrdengineapi.h +++ b/database/engine/rrdengineapi.h @@ -59,5 +59,6 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db extern int rrdeng_exit(struct rrdengine_instance *ctx); extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx); +extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t); #endif /* NETDATA_RRDENGINEAPI_H */ diff --git a/database/rrd.h b/database/rrd.h index 17ab610107..59d0501bdb 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -41,10 +41,17 @@ struct pg_cache_page_index; #include "aclk/aclk.h" #endif +enum { + CONTEXT_FLAGS_ARCHIVE = 0x01, + CONTEXT_FLAGS_CHART = 0x02, + CONTEXT_FLAGS_CONTEXT = 0x04 +}; + struct context_param { RRDDIM *rd; time_t first_entry_t; time_t last_entry_t; + uint8_t flags; }; #define META_CHART_UPDATED 1 @@ -533,7 +540,10 @@ struct rrdset { size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called - time_t last_accessed_time; // the last time this RRDSET has been accessed + union { + time_t last_accessed_time; // the last time this RRDSET has been accessed + time_t last_entry_t; // the last_entry_t computed for transient RRDSET + }; time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream char *plugin_name; // the name of the plugin that generated this diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index d73792be8b..63b0196439 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -755,7 +755,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind host parameter to fetch archived charts"); - return; + goto failed; } rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0); @@ -905,25 +905,41 @@ failed: return; } -#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname;" +#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;" +#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;" RRDHOST *sql_create_host_by_uuid(char *hostname) { int rc; RRDHOST *host = NULL; + uuid_t host_uuid; sqlite3_stmt *res = NULL; - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to fetch host"); - return NULL; + rc = uuid_parse(hostname, host_uuid); + if (!rc) { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by uuid"); + return NULL; + } + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to fetch host information"); + goto failed; + } } - - rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind hostname parameter to fetch host information"); - return NULL; + else { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by hostname"); + return NULL; + } + rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind hostname parameter to fetch host information"); + goto failed; + } } rc = sqlite3_step(res); @@ -938,13 +954,17 @@ RRDHOST *sql_create_host_by_uuid(char *hostname) host = callocz(1, sizeof(RRDHOST)); set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname, - (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, + (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5), (char *) sqlite3_column_text(res, 4), NULL, NULL); uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0))); - host->system_info = NULL; + host->system_info = callocz(1, sizeof(*host->system_info));; + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); +#ifdef ENABLE_DBENGINE + host->rrdeng_ctx = &multidb_ctx; +#endif failed: rc = sqlite3_finalize(res); @@ -1098,3 +1118,182 @@ failed: return; } + +int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id, + uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid) +{ +#ifdef ENABLE_DBENGINE + int rc; + uuid_t legacy_uuid; + uuid_t multihost_legacy_uuid; + time_t dim_first_entry_t, dim_last_entry_t; + + rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t); + if (unlikely(rc)) { + rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(rc)) { + rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(!rc)) + uuid_copy(*rrdeng_uuid, multihost_legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, *uuid); + + if (likely(!rc)) { + *first_entry_t = MIN(*first_entry_t, dim_first_entry_t); + *last_entry_t = MAX(*last_entry_t, dim_last_entry_t); + } + return rc; +#else + UNUSED(machine_guid); + UNUSED(chart_id); + UNUSED(dim_id); + UNUSED(uuid); + UNUSED(first_entry_t); + UNUSED(last_entry_t); + UNUSED(rrdeng_uuid); + return 1; +#endif +} + +#ifdef ENABLE_DBENGINE +static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid) +{ + RRDDIM *rd = callocz(1, sizeof(*rd)); + rd->rrdset = st; + rd->last_stored_value = NAN; + rrddim_flag_set(rd, RRDDIM_FLAG_NONE); + rd->state = mallocz(sizeof(*rd->state)); + rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + rd->state->query_ops.init = rrdeng_load_metric_init; + rd->state->query_ops.next_metric = rrdeng_load_metric_next; + rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished; + rd->state->query_ops.finalize = rrdeng_load_metric_finalize; + rd->state->query_ops.latest_time = rrdeng_metric_latest_time; + rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time; + rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t)); + uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid); + rd->state->metric_uuid = rd->state->rrdeng_uuid; + rd->id = strdupz(id); + rd->name = strdupz(name); + return rd; +} +#endif + +#define SELECT_CHART_CONTEXT "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \ + "order by c.chart_id asc, c.type||c.id desc;" + +#define SELECT_CHART_SINGLE "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, c.context from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \ + "order by c.chart_id asc, c.type||'.'||c.id desc;" + +void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart) +{ +#ifdef ENABLE_DBENGINE + int rc; + + if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + + if (unlikely(!(*param_list))) { + *param_list = mallocz(sizeof(struct context_param)); + (*param_list)->first_entry_t = LONG_MAX; + (*param_list)->last_entry_t = 0; + (*param_list)->rd = NULL; + (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE; + if (chart) + (*param_list)->flags |= CONTEXT_FLAGS_CHART; + else + (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT; + } + + sqlite3_stmt *res = NULL; + + if (context) + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0); + else + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host archived charts"); + return; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + if (context) + rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC); + else + rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + RRDSET *st = NULL; + char machine_guid[GUID_LEN + 1]; + uuid_unparse_lower(host->host_uuid, machine_guid); + uuid_t rrdeng_uuid; + uuid_t chart_id; + + while (sqlite3_step(res) == SQLITE_ROW) { + char id[512]; + sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1)); + + if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) { + st = callocz(1, sizeof(*st)); + char n[RRD_ID_LENGTH_MAX + 1]; + + snprintfz( + n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4), + (char *)sqlite3_column_text(res, 3)); + st->name = strdupz(n); + st->update_every = sqlite3_column_int(res, 6); + st->counter = 0; + if (chart) { + st->context = strdupz((char *)sqlite3_column_text(res, 8)); + strncpyz(st->id, chart, RRD_ID_LENGTH_MAX); + } + uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7)); + st->last_entry_t = 0; + st->rrdhost = host; + } + + if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1), + (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t, + &rrdeng_uuid))) + continue; + + st->counter++; + st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t); + + RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid); + rd->next = (*param_list)->rd; + (*param_list)->rd = rd; + } + if (likely(st && context && !st->context)) + st->context = strdupz(context); + +failed: + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading archived charts"); +#else + UNUSED(param_list); + UNUSED(host); + UNUSED(context); + UNUSED(chart); +#endif + return; +} diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index 05569ded58..d2bee75d2b 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -59,4 +59,5 @@ extern void db_unlock(void); extern void db_lock(void); extern void delete_dimension_uuid(uuid_t *dimension_uuid); extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value); +extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart); #endif //NETDATA_SQLITE_FUNCTIONS_H diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index f28eb5738b..bf311e22cf 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -5,8 +5,14 @@ #define JSON_DATES_JS 1 #define JSON_DATES_TIMESTAMP 2 -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd) { - rrdset_check_rdlock(r->st); +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list) +{ + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); + + if (should_lock) + rrdset_check_rdlock(r->st); //info("RRD2JSON(): %s: BEGIN", r->st->id); int row_annotations = 0, dates, dates_with_new = 0; diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h index 6d73a3ffe4..5c4e11371b 100644 --- a/web/api/formatters/json/json.h +++ b/web/api/formatters/json/json.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd); +extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list); #endif //NETDATA_API_FORMATTER_JSON_H diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index cf4f1099a2..1c06a8ff44 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -2,8 +2,16 @@ #include "json_wrapper.h" -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_label_key) { - rrdset_check_rdlock(r->st); +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, + struct context_param *context_param_list, char *chart_label_key) +{ + + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); + uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT)); + + if (should_lock) + rrdset_check_rdlock(r->st); long rows = rrdr_rows(r); long c, i; @@ -22,7 +30,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS sq[0] = '"'; } - rrdset_rdlock(r->st); + if (should_lock) + rrdset_rdlock(r->st); buffer_sprintf(wb, "{\n" " %sapi%s: 1,\n" " %sid%s: %s%s%s,\n" @@ -35,16 +44,17 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS " %safter%s: %u,\n" " %sdimension_names%s: [" , kq, kq - , kq, kq, sq, temp_rd?r->st->context:r->st->id, sq - , kq, kq, sq, temp_rd?r->st->context:r->st->name, sq + , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq + , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq , kq, kq, r->update_every , kq, kq, r->st->update_every - , kq, kq, (uint32_t)rrdset_first_entry_t_nolock(r->st) - , kq, kq, (uint32_t)rrdset_last_entry_t_nolock(r->st) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st)) + , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st)) , kq, kq, (uint32_t)r->before , kq, kq, (uint32_t)r->after , kq, kq); - rrdset_unlock(r->st); + if (should_lock) + rrdset_unlock(r->st); for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; @@ -89,7 +99,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_strcat(wb, "],\n"); // Composite charts - if (temp_rd) { + if (context_mode && temp_rd) { buffer_sprintf( wb, " %schart_ids%s: [", diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h index d48d5d1ae8..14662db740 100644 --- a/web/api/formatters/json_wrapper.h +++ b/web/api/formatters/json_wrapper.h @@ -5,7 +5,7 @@ #include "rrd2json.h" -extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_key); +extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key); extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index d8e2480663..82be7627c1 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -65,6 +65,7 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st) *param_list = mallocz(sizeof(struct context_param)); (*param_list)->first_entry_t = LONG_MAX; (*param_list)->last_entry_t = 0; + (*param_list)->flags = 0; (*param_list)->rd = NULL; } @@ -214,9 +215,9 @@ int rrdset2anything_api_v1( , struct context_param *context_param_list , char *chart_label_key ) { - time_t last_accessed_time = now_realtime_sec(); - st->last_accessed_time = last_accessed_time; + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) + st->last_accessed_time = now_realtime_sec(); RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list); if(!r) { @@ -238,7 +239,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); rrdr2ssv(r, wb, options, "", " ", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -251,7 +252,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV_COMMA: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); rrdr2ssv(r, wb, options, "", ",", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -264,7 +265,7 @@ int rrdset2anything_api_v1( case DATASOURCE_JS_ARRAY: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); rrdr2ssv(r, wb, options, "[", ",", "]"); rrdr_json_wrapper_end(r, wb, format, options, 0); } @@ -277,7 +278,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -290,7 +291,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_MARKDOWN: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -303,7 +304,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_JSON_ARRAY: wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) { - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); buffer_strcat(wb, "[\n"); rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); buffer_strcat(wb, "\n]"); @@ -320,7 +321,7 @@ int rrdset2anything_api_v1( case DATASOURCE_TSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -333,7 +334,7 @@ int rrdset2anything_api_v1( case DATASOURCE_HTML: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n"); rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd); buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n"); @@ -351,9 +352,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); - rrdr2json(r, wb, options, 1, temp_rd); + rrdr2json(r, wb, options, 1, context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -363,9 +364,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); - rrdr2json(r, wb, options, 1, temp_rd); + rrdr2json(r, wb, options, 1, context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -374,9 +375,9 @@ int rrdset2anything_api_v1( case DATASOURCE_JSONP: wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); - rrdr2json(r, wb, options, 0, temp_rd); + rrdr2json(r, wb, options, 0, context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -387,9 +388,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); - rrdr2json(r, wb, options, 0, temp_rd); + rrdr2json(r, wb, options, 0, context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 2733d4f2d9..2a27a94fcc 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -281,8 +281,14 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI // ---------------------------------------------------------------------------- -static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, RRDDIM *temp_rd) { - rrdset_check_rdlock(r->st); +static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, + struct context_param *context_param_list) +{ + RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); + + if (should_lock) + rrdset_check_rdlock(r->st); if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return; @@ -1060,10 +1066,11 @@ static RRDR *rrd2rrdr_fixedstep( // ------------------------------------------------------------------------- // disable the not-wanted dimensions - rrdset_check_rdlock(st); + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) + rrdset_check_rdlock(st); if(dimensions) - rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd); + rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); // ------------------------------------------------------------------------- @@ -1435,11 +1442,11 @@ static RRDR *rrd2rrdr_variablestep( // ------------------------------------------------------------------------- // disable the not-wanted dimensions - - rrdset_check_rdlock(st); + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) + rrdset_check_rdlock(st); if(dimensions) - rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd); + rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); // ------------------------------------------------------------------------- @@ -1591,8 +1598,12 @@ RRDR *rrd2rrdr( if (first_entry_t > after_requested) first_entry_t = after_requested; - if (context_param_list) + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { rebuild_context_param_list(context_param_list, after_requested); + st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; + if (unlikely(!st)) + return NULL; + } #ifdef ENABLE_DBENGINE if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index ef237fa026..9d28ae9e85 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -108,7 +108,8 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param RRDR *r = callocz(1, sizeof(RRDR)); r->st = st; - rrdr_lock_rrdset(r); + if (!context_param_list || (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))) + rrdr_lock_rrdset(r); RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; RRDDIM *rd; diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 7c5f9fb9e8..72962f9b10 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -511,6 +511,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c rrdhost_unlock(host); if (likely(context_param_list && context_param_list->rd)) // Just set the first one st = context_param_list->rd->rrdset; + else + sql_build_context_param_list(&context_param_list, host, context, NULL); } else { st = rrdset_find(host, chart); @@ -518,6 +520,17 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c st = rrdset_find_byname(host, chart); if (likely(st)) st->last_accessed_time = now_realtime_sec(); + else + sql_build_context_param_list(&context_param_list, host, NULL, chart); + } + + if (!st) { + if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset)) + st = context_param_list->rd->rrdset; + else { + free_context_param_list(&context_param_list); |