systemd-journal and facets improvements (#15926)

7 files changed, 224 insertions, 115 deletions

diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c
index 14f5c0a9bd..7e1e0313a1 100644
--- a/collectors/apps.plugin/apps_plugin.c
+++ b/collectors/apps.plugin/apps_plugin.c
@@ -4696,21 +4696,24 @@ static void function_processes(const char *transaction, char *function __maybe_u
         if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {
             category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);
             if(!category) {
-                pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
+                pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,
+                                                       "No category with that name found.");
                 return;
             }
         }
         else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
             user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
             if(!user) {
-                pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
+                pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,
+                                                       "No user with that name found.");
                 return;
             }
         }
         else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {
             group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);
             if(!group) {
-                pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
+                pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,
+                                                       "No group with that name found.");
                 return;
             }
         }
@@ -4736,7 +4739,7 @@ static void function_processes(const char *transaction, char *function __maybe_u
         else {
             char msg[PLUGINSD_LINE_MAX];
             snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
-            pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
+            pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, msg);
             return;
         }
     }
@@ -5562,7 +5565,8 @@ static void *reader_main(void *arg __maybe_unused) {
                 if(strncmp(function, "processes", strlen("processes")) == 0)
                     function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
                 else
-                    pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
+                    pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,
+                                                           "No function with this name found in apps.plugin.");
 
                 fflush(stdout);
                 netdata_mutex_unlock(&mutex);
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
index 4988b50719..135f30eca4 100644
--- a/collectors/plugins.d/plugins_d.h
+++ b/collectors/plugins.d/plugins_d.h
@@ -123,7 +123,7 @@ size_t pluginsd_initialize_plugin_directories();
 #define pluginsd_function_result_end_to_stdout() \
     fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
 
-static inline void pluginsd_function_json_error(const char *transaction, int code, const char *msg) {
+static inline void pluginsd_function_json_error_to_stdout(const char *transaction, int code, const char *msg) {
     char buffer[PLUGINSD_LINE_MAX + 1];
     json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
 
diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h
index 8635059d92..9fdb5846dd 100644
--- a/collectors/plugins.d/pluginsd_parser.h
+++ b/collectors/plugins.d/pluginsd_parser.h
@@ -10,6 +10,9 @@
 // this has to be in-sync with the same at receiver.c
 #define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
 
+// this controls the max response size of a function
+#define PLUGINSD_MAX_DEFERRED_SIZE (20 * 1024 * 1024)
+
 // PARSER return codes
 typedef enum __attribute__ ((__packed__)) parser_rc {
     PARSER_RC_OK,       // Callback was successful, go on
@@ -157,9 +160,9 @@ static inline int parser_action(PARSER *parser, char *input) {
         if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) {
             if(parser->defer.response) {
                 buffer_strcat(parser->defer.response, input);
-                if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) {
-                    // more than 10MB of data
-                    // a bad plugin that did not send the end_keyword
+                if(buffer_strlen(parser->defer.response) > PLUGINSD_MAX_DEFERRED_SIZE) {
+                    // more than PLUGINSD_MAX_DEFERRED_SIZE of data,
+                    // or a bad plugin that did not send the end_keyword
                     internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
                     return 1;
                 }
diff --git a/collectors/systemd-journal.plugin/systemd-journal.c b/collectors/systemd-journal.plugin/systemd-journal.c
index e580aa00c6..6a10f6f082 100644
--- a/collectors/systemd-journal.plugin/systemd-journal.c
+++ b/collectors/systemd-journal.plugin/systemd-journal.c
@@ -5,7 +5,7 @@
  *  GPL v3+
  */
 
-// TODO - 1) MARKDOC
+// TODO - 1) MARKDOC 2) HELP TEXT
 
 #include "collectors/all.h"
 #include "libnetdata/libnetdata.h"
@@ -31,6 +31,9 @@
 #define JOURNAL_PARAMETER_QUERY                 "query"
 #define JOURNAL_PARAMETER_FACETS                "facets"
 #define JOURNAL_PARAMETER_HISTOGRAM             "histogram"
+#define JOURNAL_PARAMETER_DIRECTION             "direction"
+#define JOURNAL_PARAMETER_IF_MODIFIED_SINCE     "if_modified_since"
+
 
 #define SYSTEMD_ALWAYS_VISIBLE_KEYS             NULL
 #define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS       NULL
@@ -39,19 +42,11 @@
     "|SYSLOG_IDENTIFIER"                        \
     "|SYSLOG_FACILITY"                          \
     "|PRIORITY"                                 \
-    "|_HOSTNAME"                                \
-    "|_RUNTIME_SCOPE"                           \
-    "|_PID"                                     \
     "|_UID"                                     \
     "|_GID"                                     \
     "|_SYSTEMD_UNIT"                            \
     "|_SYSTEMD_SLICE"                           \
-    "|_SYSTEMD_USER_SLICE"                      \
     "|_COMM"                                    \
-    "|_EXE"                                     \
-    "|_SYSTEMD_CGROUP"                          \
-    "|_SYSTEMD_USER_UNIT"                       \
-    "|USER_UNIT"                                \
     "|UNIT"                                     \
     "|CONTAINER_NAME"                           \
     "|IMAGE_NAME"                               \
@@ -65,7 +60,7 @@ DICTIONARY *gids = NULL;
 
 // ----------------------------------------------------------------------------
 
-int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t stop_monotonic_ut) {
+int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t if_modified_since, usec_t stop_monotonic_ut) {
     sd_journal *j = NULL;
     int r;
 
@@ -91,6 +86,7 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be
 
     facets_rows_begin(facets);
 
+    uint64_t first_msg_ut = 0;
     bool timed_out = false;
     size_t row_counter = 0;
     sd_journal_seek_realtime_usec(j, before_ut);
@@ -99,6 +95,16 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be
 
             uint64_t msg_ut;
             sd_journal_get_realtime_usec(j, &msg_ut);
+
+            if(unlikely(!first_msg_ut)) {
+                if(msg_ut == if_modified_since) {
+                    sd_journal_close(j);
+                    return HTTP_RESP_NOT_MODIFIED;
+                }
+
+                first_msg_ut = msg_ut;
+            }
+
             if (msg_ut < after_ut)
                 break;
 
@@ -136,6 +142,7 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be
     buffer_json_member_add_string(wb, "type", "table");
     buffer_json_member_add_time_t(wb, "update_every", 1);
     buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
+    buffer_json_member_add_uint64(wb, "last_modified", first_msg_ut);
 
     facets_report(facets, wb);
 
@@ -341,7 +348,7 @@ static void function_systemd_journal(const char *transaction, char *function, ch
     buffer_flush(wb);
     buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);
 
-    FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,
+    FACETS *facets = facets_create(50, FACETS_OPTION_ALL_KEYS_FTS,
                                    SYSTEMD_ALWAYS_VISIBLE_KEYS,
                                    SYSTEMD_KEYS_INCLUDED_IN_FACETS,
                                    SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);
@@ -349,10 +356,12 @@ static void function_systemd_journal(const char *transaction, char *function, ch
     facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);
     facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);
     facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
+    facets_accepted_param(facets, JOURNAL_PARAMETER_DIRECTION);
     facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
     facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
     facets_accepted_param(facets, JOURNAL_PARAMETER_FACETS);
     facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);
+    facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE);
 
     // register the fields in the order you want them on the dashboard
 
@@ -382,7 +391,9 @@ static void function_systemd_journal(const char *transaction, char *function, ch
 
     time_t after_s = 0, before_s = 0;
     usec_t anchor = 0;
+    usec_t if_modified_since = 0;
     size_t last = 0;
+    FACETS_ANCHOR_DIRECTION direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
     const char *query = NULL;
     const char *chart = NULL;
 
@@ -404,9 +415,15 @@ static void function_systemd_journal(const char *transaction, char *function, ch
         else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1) == 0) {
             before_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1]);
         }
+        else if(strncmp(keyword, JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":", sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1) == 0) {
+            if_modified_since = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1], NULL);
+        }
         else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1) == 0) {
             anchor = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1], NULL);
         }
+        else if(strncmp(keyword, JOURNAL_PARAMETER_DIRECTION ":", sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1) == 0) {
+            direction = strcasecmp(&keyword[sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1], "forward") ? FACETS_ANCHOR_DIRECTION_FORWARD : FACETS_ANCHOR_DIRECTION_BACKWARD;
+        }
         else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", sizeof(JOURNAL_PARAMETER_LAST ":") - 1) == 0) {
             last = str2ul(&keyword[sizeof(JOURNAL_PARAMETER_LAST ":") - 1]);
         }
@@ -483,7 +500,9 @@ static void function_systemd_journal(const char *transaction, char *function, ch
 
     buffer_json_member_add_time_t(wb, "after", after_s);
     buffer_json_member_add_time_t(wb, "before", before_s);
+    buffer_json_member_add_uint64(wb, "if_modified_since", if_modified_since);
     buffer_json_member_add_uint64(wb, "anchor", anchor);
+    buffer_json_member_add_string(wb, "direction", direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
     buffer_json_member_add_uint64(wb, "last", last);
     buffer_json_member_add_string(wb, "query", query);
     buffer_json_member_add_string(wb, "chart", chart);
@@ -491,21 +510,20 @@ static void function_systemd_journal(const char *transaction, char *function, ch
     buffer_json_object_close(wb); // request
 
     facets_set_items(facets, last);
-    facets_set_anchor(facets, anchor);
+    facets_set_anchor(facets, anchor, direction);
     facets_set_query(facets, query);
     facets_set_histogram(facets, chart ? chart : "PRIORITY", after_s * USEC_PER_SEC, before_s * USEC_PER_SEC);
 
     int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,
-                                       now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
+                                       if_modified_since, now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
 
     if(response != HTTP_RESP_OK) {
-        pluginsd_function_json_error(transaction, response, "failed");
+        pluginsd_function_json_error_to_stdout(transaction, response, "failed");
         goto cleanup;
     }
 
-    pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);
+    pluginsd_function_result_begin_to_stdout(transaction, response, "application/json", expires);
     fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
-
     pluginsd_function_result_end_to_stdout();
 
 cleanup:
@@ -545,7 +563,8 @@ static void *reader_main(void *arg __maybe_unused) {
                 if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)
                     function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
                 else
-                    pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in systemd-journal.plugin.");
+                    pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,
+                                                           "No function with this name found in systemd-journal.plugin.");
 
                 fflush(stdout);
                 netdata_mutex_unlock(&mutex);
@@ -586,7 +605,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
     // debug
 
     if(argc == 2 && strcmp(argv[1], "debug") == 0) {
-        char buf[] = "systemd-journal after:-86400 before:0 last:500";
+        char buf[] = "systemd-journal after:-864000 before:0 last:500";
         function_systemd_journal("123", buf, "", 0, 30);
         exit(1);
     }
diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c
index ea5831873a..fbcea36534 100644
--- a/libnetdata/facets/facets.c
+++ b/libnetdata/facets/facets.c
@@ -9,11 +9,17 @@ static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
 static inline void uint64_to_char(uint64_t num, char *out) {
     static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
 
-    int i;
-    for(i = 10; i >= 0; --i) {
-        out[i] = id_encoding_characters[num & 63];
-        num >>= 6;
-    }
+    out[10] = id_encoding_characters[num & 63]; num >>= 6;
+    out[9]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[8]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[7]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[6]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[5]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[4]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[3]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[2]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[1]  = id_encoding_characters[num & 63]; num >>= 6;
+    out[0]  = id_encoding_characters[num & 63];
 }
 
 inline void facets_string_hash(const char *src, size_t len, char *out) {
@@ -79,7 +85,11 @@ struct facets {
     SIMPLE_PATTERN *included_keys;
 
     FACETS_OPTIONS options;
-    usec_t anchor;
+
+    struct {
+        usec_t key;
+        FACETS_ANCHOR_DIRECTION direction;
+    } anchor;
 
     SIMPLE_PATTERN *query;          // the full text search pattern
     size_t keys_filtered_by_query;  // the number of fields we do full text search (constant)
@@ -99,7 +109,7 @@ struct facets {
         char *chart;
         bool enabled;
         uint32_t slots;
-        usec_t slot_width;
+        usec_t slot_width_ut;
         usec_t after_ut;
         usec_t before_ut;
     } histogram;
@@ -125,42 +135,51 @@ struct facets {
 
 static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) {
     // Array of valid durations in seconds
-    static time_t valid_durations[] = {
-            1,
-            15,
-            30,
+    static time_t valid_durations_s[] = {
+            1, 2, 5, 10, 15, 30,                                                // seconds
             1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60,          // minutes
             1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600,                  // hours
             1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400,  // days
             1 * (30*86400)                                                      // months
     };
-    static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
+    static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]);
 
-    usec_t duration = before_ut - after_ut;
-    usec_t bar_width = 1 * 60;
+    usec_t duration_ut = before_ut - after_ut;
+    usec_t bar_width_ut = 1 * 60 * USEC_PER_SEC;
 
     for (int i = array_size - 1; i >= 0; --i) {
-        if (duration / (valid_durations[i] * 60) >= HISTOGRAM_COLUMNS) {
-            bar_width = valid_durations[i] * 60;
+        if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= HISTOGRAM_COLUMNS) {
+            bar_width_ut = valid_durations_s[i] * USEC_PER_SEC;
             break;
         }
     }
 
-    return bar_width;
+    return bar_width_ut;
 }
 
 static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) {
-    usec_t delta = ut % facets->histogram.slot_width;
-    return ut - delta;
+    usec_t delta_ut = ut % facets->histogram.slot_width_ut;
+    return ut - delta_ut;
 }
 
 void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut) {
+    if(after_ut > before_ut) {
+        usec_t t = after_ut;
+        after_ut = before_ut;
+        before_ut = t;
+    }
+
     facets->histogram.enabled = true;
     facets->histogram.chart = chart ? strdupz(chart) : NULL;
-    facets->histogram.slot_width = calculate_histogram_bar_width(after_ut, before_ut);
+    facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut);
     facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut);
-    facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width;
-    facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width + 1;
+    facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut;
+    facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1;
+
+    if(facets->histogram.slots > 1000) {
+        facets->histogram.slots = 1000 + 1;
+        facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000;
+    }
 }
 
 static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __maybe_unused, FACET_VALUE *v, usec_t usec) {
@@ -178,7 +197,7 @@ static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __
     if(base_ut > facets->histogram.before_ut)
         base_ut = facets->histogram.before_ut;
 
-    uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width;
+    uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut;
 
     if(unlikely(slot >= facets->histogram.slots))
         slot = facets->histogram.slots - 1;
@@ -186,9 +205,12 @@ static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __
     v->histogram[slot]++;
 }
 
-static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {
     buffer_json_member_add_array(wb, key);
     {
+        if(first_key)
+            buffer_json_add_array_item_string(wb, first_key);
+
         FACET_VALUE *v;
         dfe_start_read(k->values, v) {
             if(unlikely(!v->histogram))
@@ -478,25 +500,31 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
             buffer_json_member_add_uint64(wb, "sl", 1);
             buffer_json_member_add_uint64(wb, "qr", 1);
         }
-        buffer_json_object_close(wb); // nodes;
+        buffer_json_object_close(wb); // nodes
         buffer_json_member_add_object(wb, "contexts");
         {
             buffer_json_member_add_uint64(wb, "sl", 1);
             buffer_json_member_add_uint64(wb, "qr", 1);
         }
-        buffer_json_object_close(wb); // contexts;
+        buffer_json_object_close(wb); // contexts
+        buffer_json_member_add_object(wb, "instances");
+        {
+            buffer_json_member_add_uint64(wb, "sl", 1);
+            buffer_json_member_add_uint64(wb, "qr", 1);
+        }
+        buffer_json_object_close(wb); // instances
         buffer_json_member_add_object(wb, "dimensions");
         {
             buffer_json_member_add_uint64(wb, "sl", dimensions);
             buffer_json_member_add_uint64(wb, "qr", dimensions);
         }
-        buffer_json_object_close(wb); // contexts;
+        buffer_json_object_close(wb); // dimension
     }
     buffer_json_object_close(wb); // totals
 
     buffer_json_member_add_object(wb, "result");
     {
-        facets_histogram_value_names(wb, facets, k, "labels");
+        facets_histogram_value_names(wb, facets, k, "labels", "time");
 
         buffer_json_member_add_object(wb, "point");
         {
@@ -522,8 +550,8 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
                         buffer_json_add_array_item_array(wb); // point
 
                         buffer_json_add_array_item_uint64(wb, v->histogram[i]);
-                        buffer_json_add_array_item_uint64(wb, 0);
-                        buffer_json_add_array_item_uint64(wb, 1);
+                        buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate
+                        buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation
 
                         buffer_json_array_close(wb); // point
                     }
@@ -531,7 +559,7 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
                 }
                 buffer_json_array_close(wb); // row
 
-                t += facets->histogram.slot_width;
+                t += facets->histogram.slot_width_ut;
             }
         }
         buffer_json_array_close(wb); //data
@@ -541,13 +569,13 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
     buffer_json_member_add_object(wb, "db");
     {
         buffer_json_member_add_uint64(wb, "tiers", 1);
-        buffer_json_member_add_uint64(wb, "update_every", 1);
+        buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
         buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
         buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
         buffer_json_member_add_string(wb, "units", "events");
         buffer_json_member_add_object(wb, "dimensions");
         {
-            facets_histogram_value_names(wb, facets, k, "ids");
+            facets_histogram_value_names(wb, facets, k, "ids", NULL);
             facets_histogram_value_units(wb, facets, k, "units");
 
             buffer_json_member_add_object(wb, "sts");
@@ -569,7 +597,7 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
                 buffer_json_member_add_uint64(wb, "tier", 0);
                 buffer_json_member_add_uint64(wb, "queries", 1);
                 buffer_json_member_add_uint64(wb, "points", count);
-                buffer_json_member_add_time_t(wb, "update_every", 1);
+                buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
                 buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
                 buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
             }
@@ -582,11 +610,11 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
     buffer_json_member_add_object(wb, "view");
     {
         buffer_json_member_add_string(wb, "title", "Events Distribution");
-        buffer_json_member_add_time_t(wb, "update_every", 1);
+        buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
         buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);
         buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);
         buffer_json_member_add_string(wb, "units", "events");
-        buffer_json_member_add_string(wb, "chart_type", "stacked");
+        buffer_json_member_add_string(wb, "chart_type", "stackedBar");
         buffer_json_member_add_object(wb, "dimensions");
         {
             buffer_json_member_add_array(wb, "grouped_by");
@@ -595,8 +623,8 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)
             }
             buffer_json_array_close(wb); // grouped_by
 
-            facets_histogram_value_names(wb, facets, k, "ids");
-            facets_histogram_value_names(wb, facets, k, "names");
+            facets_histogram_value_names(wb, facets, k, "ids", NULL);
+            facets_histogram_value_names(wb, facets, k, "names", NULL);
             facets_histogram_value_units(wb, facets, k, "units");
 
             buffer_json_member_add_object(wb, "sts");
@@ -792,7 +820,7 @@ static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused
 
 // ----------------------------------------------------------------------------
 
-FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
+FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
     FACETS *facets = callocz(1, sizeof(FACETS));
     facets->options = options;
     facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY));
@@ -810,7 +838,8 @@ FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS op
         facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
 
     facets->max_items_to_return = items_to_return;
-    facets->anchor = anchor;
+    facets->anchor.key = 0;
+    facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
     facets->order = 1;
 
     return facets;
@@ -877,8 +906,9 @@ void facets_set_items(FACETS *facets, uint32_t items) {
     facets->max_items_to_return = items;
 }
 
-void facets_set_anchor(FACETS *facets, usec_t anchor) {
-    facets->anchor = anchor;
+void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction) {
+    facets->anchor.key = anchor;
+    facets->anchor.direction = direction;
 }
 
 inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) {
@@ -1027,24 +1057,12 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into
 
 // ----------------------------------------------------------------------------
 
-static void facets_row_keep(FACETS *facets, usec_t usec) {
-    facets->operations.matched++;
+static inline FACET_ROW *facets_row_keep_seek_to_position(FACETS *facets, usec_t usec) {
+    if(usec < facets->base->prev->usec)
+        return facets->base->prev;
 
-    if(usec < facets->anchor) {
-        facets->operations.skips_before++;
-        return;
-    }
-
-    if(unlikely(!facets->base)) {
-        facets->operations.last_added = facets_row_create(facets, usec, NULL);
-        DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
-        facets->items_to_return++;
-        facets->operations.first++;
-        return;
-    }
-
-    if(likely(usec > facets->base->prev->usec))
-        facets->operations.last_added = facets->base->prev;
+    if(usec > facets->base->usec)
+        return facets->base;
 
     FACET_ROW *last = facets->operations.last_added;
     while(last->prev != facets->base->prev && usec > last->prev->usec) {
@@ -1057,45 +1075,104 @@ static void facets_row_keep(FACETS *facets, usec_t usec) {
         facets->operations.forwards++;
     }
 
-    if(facets->items_to_return >= facets->max_items_to_return) {
-        if(last == facets->base->prev && usec < last->usec) {
-            facets->operations.skips_after++;
-            return;
-        }
-    }
+    return last;
+}
 
+static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) {
+    facets->operations.last_added = facets_row_create(facets, usec, NULL);
+    DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
     facets->items_to_return++;
+    facets->operations.first++;
+}
+
+static void facets_row_keep(FACETS *facets, usec_t usec) {
+    facets->operations.matched++;
+
+    if(facets->anchor.key) {
+        // we have an anchor key
+        // we don't want to keep rows on the other side of the direction
 
-    if(usec > last->usec) {
-        if(facets->items_to_return > facets->max_items_to_return) {
-            facets->items_to_return--;
-            facets->operations.shifts++;
-            facets->operations.last_added = facets->base->prev;
-            DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
-            facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added);
+        switch (facets->anchor.direction) {
+            default:
+            case FACETS_ANCHOR_DIRECTION_BACKWARD:
+                if (usec < facets->anchor.key) {
+                    facets->operations.skips_before++;
+                    return;
+                }
+                break;
+
+            case FACETS_ANCHOR_DIRECTION_FORWARD:
+                if (usec > facets->anchor.key) {
+                    facets->operations.skips_after++;
+                    return;
+                }
+                break;
         }
-        DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
-        facets->operations.prepends++;
     }
-    else {
-        facets->operations.last_added = facets_row_create(facets, usec, NULL);
-        DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
-        facets->operations.appends++;
+
+    if(unlikely(!facets->base)) {
+        // the first row to keep
+        facets_row_keep_first_entry(facets, usec);
+        return;
     }
 
-    while(facets->items_to_return > facets->max_items_to_return) {
-        // we have to remove something
+    FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec);
+    FACET_ROW *to_replace = NULL;
 
-        FACET_ROW *tmp = facets->base->prev;
-        DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next);
-        facets->items_to_return--;
+    if(likely(facets->items_to_return >= facets->max_items_to_return)) {
+        // we have enough items to return already
+
+        switch(facets->anchor.direction) {
+            default:
+            case FACETS_ANCHOR_DIRECTION_BACKWARD:
+                if(closest == facets->base->prev && usec < closest->usec) {
+                    // this is to the end of the list, belonging to the next page
+                    facets->operations.skips_after++;
+                    return;
+                }
 
-        if(unlikely(facets->operations.last_added == tmp))
-            facets->operations.last_added = facets->base->prev;
+                // it seems we need to remove an item - the last one
+                to_replace = facets->base->prev;
+                if(closest == to_replace)
+                    closest = to_replace->prev;
+
+                break;
+
+            case FACETS_ANCHOR_DIRECTION_FORWARD:
+                if(closest == facets->base && usec > closest->usec) {
+                    // this is to the beginning of the list, belonging to the next page
+                    facets->operations.skips_before++;
+                    return;
+                }
+
+                // it seems we need to remove an item - the first one
+                to_replace = facets->base;
+                if(closest == to_replace)
+                    closest = to_replace->next;
+
+                break;
+        }
 
-        facets_row_free(facets, tmp);
         facets->operations.shifts++;
+        facets->items_to_return--;
+        DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next);
+    }
+
+    internal_fatal(!closest, "FACETS: closest cannot be NULL");
+    internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace");
+
+    facets->operations.last_added = facets_row_create(facets, usec, to_replace);
+
+    if(usec < closest->usec) {
+        DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
+        facets->operations.appends++;
     }
+    else {
+        DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
+        facets->operations.prepends++;
+    }
+
+    facets->items_to_return++;
 }
 
 void facets_rows_begin(FACETS *facets) {
diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h
index 2933a020fb..97a664d8b7 100644
--- a/libnetdata/facets/facets.h
+++ b/libnetdata/facets/facets.h
@@ -6,6 +6,11 @@
 #define FACET_VALUE_UNSET "-"
 
 typedef enum __attribute__((packed)) {
+    FACETS_ANCHOR_DIRECTION_FORWARD,
+    FACETS_ANCHOR_DIRECTION_BACKWARD,
+} FACETS_ANCHOR_DIRECTION;
+
+typedef enum __attribute__((packed)) {
     FACET_KEY_OPTION_FACET      = (1 << 0), // filterable values
     FACET_KEY_OPTION_NO_FACET   = (1 << 1), // non-filterable value
     FACET_KEY_OPTION_STICKY     = (1 << 2), // should be sticky in the table
@@ -42,7 +47,7 @@