Add data query support for archived charts (#10771)

14 files changed, 345 insertions, 71 deletions

diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index 7b2ff5b721..cb46e06e3b 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -691,6 +691,36 @@ time_t rrdeng_metric_oldest_time(RRDDIM *rd)
     return page_index->oldest_time / USEC_PER_SEC;
 }
 
+int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t)
+{
+    struct page_cache *pg_cache;
+    struct rrdengine_instance *ctx;
+    Pvoid_t *PValue;
+    struct pg_cache_page_index *page_index = NULL;
+
+    ctx = get_rrdeng_ctx_from_host(localhost);
+    if (unlikely(!ctx)) {
+        error("Failed to fetch multidb context");
+        return 1;
+    }
+    pg_cache = &ctx->pg_cache;
+
+    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+    PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t));
+    if (likely(NULL != PValue)) {
+        page_index = *PValue;
+    }
+    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+
+    if (likely(page_index)) {
+        *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
+        *last_entry_t = page_index->latest_time / USEC_PER_SEC;
+        return 0;
+    }
+
+    return 1;
+}
+
 /* Also gets a reference for the page */
 void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrdeng_page_descr **ret_descr)
 {
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index 41375b980c..00e55e6625 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -59,5 +59,6 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db
 
 extern int rrdeng_exit(struct rrdengine_instance *ctx);
 extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
+extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
 
 #endif /* NETDATA_RRDENGINEAPI_H */
diff --git a/database/rrd.h b/database/rrd.h
index 17ab610107..59d0501bdb 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -41,10 +41,17 @@ struct pg_cache_page_index;
 #include "aclk/aclk.h"
 #endif
 
+enum {
+    CONTEXT_FLAGS_ARCHIVE = 0x01,
+    CONTEXT_FLAGS_CHART   = 0x02,
+    CONTEXT_FLAGS_CONTEXT = 0x04
+};
+
 struct context_param {
     RRDDIM *rd;
     time_t first_entry_t;
     time_t last_entry_t;
+    uint8_t flags;
 };
 
 #define META_CHART_UPDATED 1
@@ -533,7 +540,10 @@ struct rrdset {
     size_t counter;                                 // the number of times we added values to this database
     size_t counter_done;                            // the number of times rrdset_done() has been called
 
-    time_t last_accessed_time;                      // the last time this RRDSET has been accessed
+    union {
+        time_t last_accessed_time;                  // the last time this RRDSET has been accessed
+        time_t last_entry_t;                        // the last_entry_t computed for transient RRDSET
+    };
     time_t upstream_resync_time;                    // the timestamp up to which we should resync clock upstream
 
     char *plugin_name;                              // the name of the plugin that generated this
diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c
index d73792be8b..63b0196439 100644
--- a/database/sqlite/sqlite_functions.c
+++ b/database/sqlite/sqlite_functions.c
@@ -755,7 +755,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
     rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host parameter to fetch archived charts");
-        return;
+        goto failed;
     }
 
     rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0);
@@ -905,25 +905,41 @@ failed:
     return;
 }
 
-#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname;"
+#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;"
+#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;"
 
 RRDHOST *sql_create_host_by_uuid(char *hostname)
 {
     int rc;
     RRDHOST *host = NULL;
+    uuid_t host_uuid;
 
     sqlite3_stmt *res = NULL;
 
-    rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to fetch host");
-        return NULL;
+    rc = uuid_parse(hostname, host_uuid);
+    if (!rc) {
+        rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to prepare statement to fetch host by uuid");
+            return NULL;
+        }
+        rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to bind host_id parameter to fetch host information");
+            goto failed;
+        }
     }
-
-    rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind hostname parameter to fetch host information");
-        return NULL;
+    else {
+        rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to prepare statement to fetch host by hostname");
+            return NULL;
+        }
+        rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to bind hostname parameter to fetch host information");
+            goto failed;
+        }
     }
 
     rc = sqlite3_step(res);
@@ -938,13 +954,17 @@ RRDHOST *sql_create_host_by_uuid(char *hostname)
     host = callocz(1, sizeof(RRDHOST));
 
     set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname,
-                            (char *) sqlite3_column_text(res, 1), (const char *) uuid_str,
+                        (char *) sqlite3_column_text(res, 1), (const char *) uuid_str,
                         (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5),
                         (char *) sqlite3_column_text(res, 4), NULL, NULL);
 
     uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
 
-    host->system_info = NULL;
+    host->system_info = callocz(1, sizeof(*host->system_info));;
+    rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
+#ifdef ENABLE_DBENGINE
+    host->rrdeng_ctx = &multidb_ctx;
+#endif
 
 failed:
     rc = sqlite3_finalize(res);
@@ -1098,3 +1118,182 @@ failed:
 
     return;
 }
+
+int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,
+                                uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid)
+{
+#ifdef ENABLE_DBENGINE
+    int rc;
+    uuid_t  legacy_uuid;
+    uuid_t  multihost_legacy_uuid;
+    time_t dim_first_entry_t, dim_last_entry_t;
+
+    rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t);
+    if (unlikely(rc)) {
+        rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid);
+        rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
+        if (likely(rc)) {
+            rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid);
+            rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t);
+            if (likely(!rc))
+                uuid_copy(*rrdeng_uuid, multihost_legacy_uuid);
+        }
+        else
+            uuid_copy(*rrdeng_uuid, legacy_uuid);
+    }
+    else
+        uuid_copy(*rrdeng_uuid, *uuid);
+
+    if (likely(!rc)) {
+        *first_entry_t = MIN(*first_entry_t, dim_first_entry_t);
+        *last_entry_t = MAX(*last_entry_t, dim_last_entry_t);
+    }
+    return rc;
+#else
+    UNUSED(machine_guid);
+    UNUSED(chart_id);
+    UNUSED(dim_id);
+    UNUSED(uuid);
+    UNUSED(first_entry_t);
+    UNUSED(last_entry_t);
+    UNUSED(rrdeng_uuid);
+    return 1;
+#endif
+}
+
+#ifdef ENABLE_DBENGINE
+static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
+{
+    RRDDIM *rd = callocz(1, sizeof(*rd));
+    rd->rrdset = st;
+    rd->last_stored_value = NAN;
+    rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
+    rd->state = mallocz(sizeof(*rd->state));
+    rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
+    rd->state->query_ops.init = rrdeng_load_metric_init;
+    rd->state->query_ops.next_metric = rrdeng_load_metric_next;
+    rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
+    rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
+    rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
+    rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
+    rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t));
+    uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid);
+    rd->state->metric_uuid = rd->state->rrdeng_uuid;
+    rd->id = strdupz(id);
+    rd->name = strdupz(name);
+    return rd;
+}
+#endif
+
+#define SELECT_CHART_CONTEXT  "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id from chart c, " \
+    "dimension d, host h " \
+    "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \
+    "order by c.chart_id asc, c.type||c.id desc;"
+
+#define SELECT_CHART_SINGLE  "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, c.context from chart c, " \
+    "dimension d, host h " \
+    "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
+    "order by c.chart_id asc, c.type||'.'||c.id desc;"
+
+void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart)
+{
+#ifdef ENABLE_DBENGINE
+    int rc;
+
+    if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+        return;
+
+    if (unlikely(!(*param_list))) {
+        *param_list = mallocz(sizeof(struct context_param));
+        (*param_list)->first_entry_t = LONG_MAX;
+        (*param_list)->last_entry_t = 0;
+        (*param_list)->rd = NULL;
+        (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE;
+        if (chart)
+            (*param_list)->flags |= CONTEXT_FLAGS_CHART;
+        else
+            (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT;
+    }
+
+    sqlite3_stmt *res = NULL;
+
+    if (context)
+        rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0);
+    else
+        rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to prepare statement to fetch host archived charts");
+        return;
+    }
+
+    rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind host parameter to fetch archived charts");
+        goto failed;
+    }
+
+    if (context)
+        rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC);
+    else
+        rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind host parameter to fetch archived charts");
+        goto failed;
+    }
+
+    RRDSET *st = NULL;
+    char machine_guid[GUID_LEN + 1];
+    uuid_unparse_lower(host->host_uuid, machine_guid);
+    uuid_t rrdeng_uuid;
+    uuid_t chart_id;
+
+    while (sqlite3_step(res) == SQLITE_ROW) {
+        char id[512];
+        sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1));
+
+        if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
+            st = callocz(1, sizeof(*st));
+            char n[RRD_ID_LENGTH_MAX + 1];
+
+            snprintfz(
+                n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
+                (char *)sqlite3_column_text(res, 3));
+            st->name = strdupz(n);
+            st->update_every = sqlite3_column_int(res, 6);
+            st->counter = 0;
+            if (chart) {
+                st->context = strdupz((char *)sqlite3_column_text(res, 8));
+                strncpyz(st->id, chart, RRD_ID_LENGTH_MAX);
+            }
+            uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
+            st->last_entry_t = 0;
+            st->rrdhost = host;
+        }
+
+        if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1),
+                (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
+                &rrdeng_uuid)))
+            continue;
+
+        st->counter++;
+        st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
+
+        RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
+        rd->next = (*param_list)->rd;
+        (*param_list)->rd = rd;
+    }
+    if (likely(st && context && !st->context))
+        st->context = strdupz(context);
+
+failed:
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement when reading archived charts");
+#else
+    UNUSED(param_list);
+    UNUSED(host);
+    UNUSED(context);
+    UNUSED(chart);
+#endif
+    return;
+}
diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h
index 05569ded58..d2bee75d2b 100644
--- a/database/sqlite/sqlite_functions.h
+++ b/database/sqlite/sqlite_functions.h
@@ -59,4 +59,5 @@ extern void db_unlock(void);
 extern void db_lock(void);
 extern void delete_dimension_uuid(uuid_t *dimension_uuid);
 extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value);
+extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart);
 #endif //NETDATA_SQLITE_FUNCTIONS_H
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index f28eb5738b..bf311e22cf 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -5,8 +5,14 @@
 #define JSON_DATES_JS 1
 #define JSON_DATES_TIMESTAMP 2
 
-void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd) {
-    rrdset_check_rdlock(r->st);
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct context_param *context_param_list)
+{
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+
+    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
+
+    if (should_lock)
+        rrdset_check_rdlock(r->st);
 
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
     int row_annotations = 0, dates, dates_with_new = 0;
diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h
index 6d73a3ffe4..5c4e11371b 100644
--- a/web/api/formatters/json/json.h
+++ b/web/api/formatters/json/json.h
@@ -5,6 +5,6 @@
 
 #include "../rrd2json.h"
 
-extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, RRDDIM *temp_rd);
+extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list);
 
 #endif //NETDATA_API_FORMATTER_JSON_H
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index cf4f1099a2..1c06a8ff44 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -2,8 +2,16 @@
 
 #include "json_wrapper.h"
 
-void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_label_key) {
-    rrdset_check_rdlock(r->st);
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
+                             struct context_param *context_param_list, char *chart_label_key)
+{
+
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
+    uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT));
+
+    if (should_lock)
+        rrdset_check_rdlock(r->st);
 
     long rows = rrdr_rows(r);
     long c, i;
@@ -22,7 +30,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
         sq[0] = '"';
     }
 
-    rrdset_rdlock(r->st);
+    if (should_lock)
+        rrdset_rdlock(r->st);
     buffer_sprintf(wb, "{\n"
                        "   %sapi%s: 1,\n"
                        "   %sid%s: %s%s%s,\n"
@@ -35,16 +44,17 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %safter%s: %u,\n"
                        "   %sdimension_names%s: ["
                    , kq, kq
-                   , kq, kq, sq, temp_rd?r->st->context:r->st->id, sq
-                   , kq, kq, sq, temp_rd?r->st->context:r->st->name, sq
+                   , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
+                   , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
                    , kq, kq, r->update_every
                    , kq, kq, r->st->update_every
-                   , kq, kq, (uint32_t)rrdset_first_entry_t_nolock(r->st)
-                   , kq, kq, (uint32_t)rrdset_last_entry_t_nolock(r->st)
+                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t(r->st))
+                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t(r->st))
                    , kq, kq, (uint32_t)r->before
                    , kq, kq, (uint32_t)r->after
                    , kq, kq);
-    rrdset_unlock(r->st);
+    if (should_lock)
+        rrdset_unlock(r->st);
 
     for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
@@ -89,7 +99,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
     buffer_strcat(wb, "],\n");
 
     // Composite charts
-    if (temp_rd) {
+    if (context_mode && temp_rd) {
         buffer_sprintf(
             wb,
             "   %schart_ids%s: [",
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index d48d5d1ae8..14662db740 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -5,7 +5,7 @@
 
 #include "rrd2json.h"
 
-extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, RRDDIM *temp_rd, char *chart_key);
+extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
 extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index d8e2480663..82be7627c1 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -65,6 +65,7 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st)
         *param_list = mallocz(sizeof(struct context_param));
         (*param_list)->first_entry_t = LONG_MAX;
         (*param_list)->last_entry_t = 0;
+        (*param_list)->flags = 0;
         (*param_list)->rd = NULL;
     }
 
@@ -214,9 +215,9 @@ int rrdset2anything_api_v1(
         , struct context_param *context_param_list
         , char *chart_label_key
 ) {
-    time_t last_accessed_time = now_realtime_sec();
-    st->last_accessed_time = last_accessed_time;
 
+    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
+        st->last_accessed_time = now_realtime_sec();
 
     RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
     if(!r) {
@@ -238,7 +239,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             rrdr2ssv(r, wb, options, "", " ", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -251,7 +252,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             rrdr2ssv(r, wb, options, "", ",", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -264,7 +265,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
             rrdr2ssv(r, wb, options, "[", ",", "]");
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
@@ -277,7 +278,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -290,7 +291,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -303,7 +304,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
             buffer_strcat(wb, "[\n");
             rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
@@ -320,7 +321,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -333,7 +334,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
             rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -351,9 +352,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
 
-        rrdr2json(r, wb, options, 1, temp_rd);
+        rrdr2json(r, wb, options, 1, context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -363,9 +364,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
 
-        rrdr2json(r, wb, options, 1, temp_rd);
+        rrdr2json(r, wb, options, 1, context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -374,9 +375,9 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
 
-        rrdr2json(r, wb, options, 0, temp_rd);
+        rrdr2json(r, wb, options, 0, context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -387,9 +388,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, temp_rd, chart_label_key);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
 
-        rrdr2json(r, wb, options, 0, temp_rd);
+        rrdr2json(r, wb, options, 0, context_param_list);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 2733d4f2d9..2a27a94fcc 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -281,8 +281,14 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI
 
 // ----------------------------------------------------------------------------
 
-static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, RRDDIM *temp_rd) {
-    rrdset_check_rdlock(r->st);
+static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
+                                                 struct context_param *context_param_list)
+{
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
+
+    if (should_lock)
+        rrdset_check_rdlock(r->st);
 
     if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
 
@@ -1060,10 +1066,11 @@ static RRDR *rrd2rrdr_fixedstep(
     // -------------------------------------------------------------------------
     // disable the not-wanted dimensions
 
-    rrdset_check_rdlock(st);
+    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
+        rrdset_check_rdlock(st);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd);
+        rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
 
 
     // -------------------------------------------------------------------------
@@ -1435,11 +1442,11 @@ static RRDR *rrd2rrdr_variablestep(
 
     // -------------------------------------------------------------------------
     // disable the not-wanted dimensions
-
-    rrdset_check_rdlock(st);
+    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
+        rrdset_check_rdlock(st);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, options, dimensions, temp_rd);
+        rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
 
 
     // -------------------------------------------------------------------------
@@ -1591,8 +1598,12 @@ RRDR *rrd2rrdr(
         if (first_entry_t > after_requested)
             first_entry_t = after_requested;
 
-    if (context_param_list)
+    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
         rebuild_context_param_list(context_param_list, after_requested);
+        st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
+        if (unlikely(!st))
+            return NULL;
+    }
 
 #ifdef ENABLE_DBENGINE
     if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c
index ef237fa026..9d28ae9e85 100644
--- a/web/api/queries/rrdr.c
+++ b/web/api/queries/rrdr.c
@@ -108,7 +108,8 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param
     RRDR *r = callocz(1, sizeof(RRDR));
     r->st = st;
 
-    rrdr_lock_rrdset(r);
+    if (!context_param_list || (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)))
+        rrdr_lock_rrdset(r);
 
     RRDDIM *temp_rd =  context_param_list ? context_param_list->rd : NULL;
     RRDDIM *rd;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index 7c5f9fb9e8..72962f9b10 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -511,6 +511,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         rrdhost_unlock(host);
         if (likely(context_param_list && context_param_list->rd))  // Just set the first one
             st = context_param_list->rd->rrdset;
+        else
+            sql_build_context_param_list(&context_param_list, host, context, NULL);
     }
     else {
         st = rrdset_find(host, chart);
@@ -518,6 +520,17 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
             st = rrdset_find_byname(host, chart);
         if (likely(st))
             st->last_accessed_time = now_realtime_sec();
+        else
+            sql_build_context_param_list(&context_param_list, host, NULL, chart);
+    }
+
+    if (!st) {
+        if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset))
+            st = context_param_list->rd->rrdset;
+        else {
+            free_context_param_list(&context_param_list);
+            context_param_list = NULL;
+        }
     }
 
     if (!st && !context_param_list) {
diff --git a/web/server/web_client.c b/web/server/web_client.c
index 30c9e5c6b3..5e3de38d5b 100644
--- a/web/server/web_client.c
+++ b/