diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2023-09-12 18:19:47 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-12 18:19:47 +0300 |
commit | bb85c57a39674be5fa7d815734aead46fb47a08c (patch) | |
tree | 9f99f5a9b990b4d4e3f9fa5adc5877e84d8335b5 /libnetdata | |
parent | 21539122d57536bd30bf92d9e62fa6cb25142c85 (diff) |
facets optimizations (#15940)
Diffstat (limited to 'libnetdata')
-rw-r--r-- | libnetdata/clocks/clocks.h | 8 | ||||
-rw-r--r-- | libnetdata/facets/facets.c | 183 | ||||
-rw-r--r-- | libnetdata/facets/facets.h | 18 |
3 files changed, 166 insertions, 43 deletions
diff --git a/libnetdata/clocks/clocks.h b/libnetdata/clocks/clocks.h index b050b62548..5b88a45798 100644 --- a/libnetdata/clocks/clocks.h +++ b/libnetdata/clocks/clocks.h @@ -16,10 +16,10 @@ struct timespec { typedef int clockid_t; #endif -typedef unsigned long long nsec_t; -typedef unsigned long long msec_t; -typedef unsigned long long usec_t; -typedef long long susec_t; +typedef uint64_t nsec_t; +typedef uint64_t msec_t; +typedef uint64_t usec_t; +typedef int64_t susec_t; typedef struct heartbeat { usec_t realtime; diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c index e9be71417f..7496072c78 100644 --- a/libnetdata/facets/facets.c +++ b/libnetdata/facets/facets.c @@ -31,12 +31,20 @@ inline void facets_string_hash(const char *src, size_t len, char *out) { out[FACET_STRING_HASH_SIZE - 1] = '\0'; } +static inline void facets_zero_hash(char *out) { + for(int i = 0; i < FACET_STRING_HASH_SIZE ;i++) + out[i] = '0'; + + out[FACET_STRING_HASH_SIZE - 1] = '\0'; +} + // ---------------------------------------------------------------------------- typedef struct facet_value { const char *name; bool selected; + bool empty; uint32_t rows_matching_facet_value; uint32_t final_facet_value_counter; @@ -46,6 +54,8 @@ typedef struct facet_value { } FACET_VALUE; struct facet_key { + FACETS *facets; + const char *name; DICTIONARY *values; @@ -61,9 +71,14 @@ struct facet_key { struct { char hash[FACET_STRING_HASH_SIZE]; bool updated; + bool empty; BUFFER *b; } current_value; + struct { + FACET_VALUE *empty_value; + } empty_value; + uint32_t order; struct { @@ -117,9 +132,6 @@ struct facets { struct { FACET_ROW *last_added; - size_t evaluated; - size_t matched; - size_t first; size_t forwards; size_t backwards; @@ -128,6 +140,32 @@ struct facets { size_t prepends; size_t appends; size_t shifts; + + struct { + size_t evaluated; + size_t matched; + size_t created; + size_t reused; + } rows; + + struct { + size_t registered; + size_t unique; + } keys; + + struct { + size_t registered; + size_t transformed; + size_t dynamic; + size_t empty; + size_t indexed; + size_t inserts; + size_t conflicts; + } values; + + struct { + size_t searches; + } fts; } operations; }; @@ -673,12 +711,12 @@ static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { bool included = true, excluded = false; - if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) { + if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { if(k->options & FACET_KEY_OPTION_FACET) { included = true; excluded = false; } - else if(k->options & FACET_KEY_OPTION_NO_FACET) { + else if(k->options & (FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { included = false; excluded = true; } @@ -721,6 +759,8 @@ static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unus v->name = strdupz(v->name); facet_value_is_used(k, v); } + + k->facets->operations.values.inserts++; } static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { @@ -738,6 +778,8 @@ static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_un internal_fatal(v->name && nv->name && strcmp(v->name, nv->name) != 0, "value hash conflict: '%s' and '%s' have the same hash '%s'", v->name, nv->name, dictionary_acquired_item_name(item)); + k->facets->operations.values.conflicts++; + return false; } @@ -768,6 +810,8 @@ static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused FACET_KEY *k = value; FACETS *facets = data; + k->facets = facets; + if(!(k->options & FACET_KEY_OPTION_REORDER)) k->order = facets->order++; @@ -783,6 +827,9 @@ static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused k->current_value.b = buffer_create(0, NULL); DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys_ll, k, prev, next); + + facets->operations.keys.registered++; + facets->operations.keys.unique++; } static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { @@ -804,6 +851,8 @@ static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus k->options &= ~FACET_KEY_OPTION_REORDER; } + facets->operations.keys.registered++; + return false; } @@ -870,17 +919,21 @@ void facets_accepted_param(FACETS *facets, const char *param) { dictionary_set(facets->accepted_params, param, NULL, 0); } -inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { +static inline FACET_KEY *facets_register_key_name_length(FACETS *facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) { FACET_KEY tk = { .name = key, .options = options, .default_selected_for_values = true, }; char hash[FACET_STRING_HASH_SIZE]; - facets_string_hash(tk.name, strlen(key), hash); + facets_string_hash(tk.name, key_length, hash); return dictionary_set(facets->keys, hash, &tk, sizeof(tk)); } +inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { + return facets_register_key_name_length(facets, key, strlen(key), options); +} + inline FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) { FACET_KEY *k = facets_register_key_name(facets, key, options); k->transform.cb = cb; @@ -938,32 +991,60 @@ void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *v // ---------------------------------------------------------------------------- static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) { + facets->operations.values.registered++; + if(!k->current_value.updated) buffer_flush(k->current_value.b); - if(k->transform.cb) + if(k->transform.cb) { + facets->operations.values.transformed++; k->transform.cb(facets, k->current_value.b, k->transform.data); + } if(!k->current_value.updated) { buffer_fast_strcat(k->current_value.b, FACET_VALUE_UNSET, sizeof(FACET_VALUE_UNSET) - 1); k->current_value.updated = true; + k->current_value.empty = true; + facets->operations.values.empty++; } + else + k->current_value.empty = false; // bool found = false; // if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL) // found = true; - if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + if(facets->query && !k->current_value.empty && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + facets->operations.fts.searches++; if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value.b))) facets->keys_matched_by_query++; } if(k->values) { - FACET_VALUE tk = { - .name = buffer_tostring(k->current_value.b), - }; - facets_string_hash(tk.name, buffer_strlen(k->current_value.b), k->current_value.hash); - dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk)); + if(k->current_value.empty) { + facets_zero_hash(k->current_value.hash); + + FACET_VALUE tk = { + .name = FACET_VALUE_UNSET, + }; + + if(k->empty_value.empty_value) { + facet_value_conflict_callback(NULL, k->empty_value.empty_value, &tk, k); + } + else { + FACET_VALUE *tkv = dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk)); + tkv->empty = true; + k->empty_value.empty_value = tkv; + } + } + else { + FACET_VALUE tk = { + .name = buffer_tostring(k->current_value.b), + }; + facets_string_hash(tk.name, buffer_strlen(k->current_value.b), k->current_value.hash); + dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk)); + facets->operations.values.indexed++; + } } else { k->key_found_in_row++; @@ -980,8 +1061,8 @@ void facets_add_key_value(FACETS *facets, const char *key, const char *value) { facets_check_value(facets, k); } -void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) { - FACET_KEY *k = facets_register_key_name(facets, key, 0); +void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) { + FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0); buffer_flush(k->current_value.b); buffer_strncat(k->current_value.b, value, value_len); k->current_value.updated = true; @@ -997,7 +1078,7 @@ static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __ma FACET_ROW *row = data; (void)row; rkv->wb = buffer_create(0, NULL); - buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET); + buffer_strcat(rkv->wb, rkv->tmp); } static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { @@ -1006,7 +1087,7 @@ static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __ FACET_ROW *row = data; (void)row; buffer_flush(rkv->wb); - buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET); + buffer_strcat(rkv->wb, n_rkv->tmp); return false; } @@ -1029,14 +1110,17 @@ static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) { FACET_ROW *row; - if(into) + if(into) { row = into; + facets->operations.rows.reused++; + } else { row = callocz(1, sizeof(FACET_ROW)); row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE)); dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row); dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row); dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row); + facets->operations.rows.created++; } row->usec = usec; @@ -1044,9 +1128,9 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into FACET_KEY *k; dfe_start_read(facets->keys, k) { FACET_ROW_KEY_VALUE t = { - .tmp = (k->current_value.updated && buffer_strlen(k->current_value.b)) ? - buffer_tostring(k->current_value.b) : FACET_VALUE_UNSET, + .tmp = (k->current_value.updated) ? buffer_tostring(k->current_value.b) : FACET_VALUE_UNSET, .wb = NULL, + .empty = k->current_value.empty, }; dictionary_set(row->dict, k->name, &t, sizeof(t)); } @@ -1086,7 +1170,7 @@ static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) { } static void facets_row_keep(FACETS *facets, usec_t usec) { - facets->operations.matched++; + facets->operations.rows.matched++; if(facets->anchor.key) { // we have an anchor key @@ -1095,14 +1179,16 @@ static void facets_row_keep(FACETS *facets, usec_t usec) { switch (facets->anchor.direction) { default: case FACETS_ANCHOR_DIRECTION_BACKWARD: - if (usec < facets->anchor.key) { + // we need to keep only the smaller timestamps + if (usec >= facets->anchor.key) { facets->operations.skips_before++; return; } break; case FACETS_ANCHOR_DIRECTION_FORWARD: - if (usec > facets->anchor.key) { + // we need to keep only the bigger timestamps + if (usec <= facets->anchor.key) { facets->operations.skips_after++; return; } @@ -1182,6 +1268,7 @@ void facets_rows_begin(FACETS *facets) { k->key_found_in_row = 0; k->key_values_selected_in_row = 0; k->current_value.updated = false; + k->current_value.empty = false; k->current_value.hash[0] = '\0'; } // dfe_done(k); @@ -1193,7 +1280,7 @@ void facets_row_finished(FACETS *facets, usec_t usec) { if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query) goto cleanup; - facets->operations.evaluated++; + facets->operations.rows.evaluated++; uint32_t total_keys = 0; uint32_t selected_by = 0; @@ -1375,7 +1462,7 @@ void facets_report(FACETS *facets, BUFFER *wb) { RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_COUNT, - k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, options, FACET_VALUE_UNSET); } @@ -1399,9 +1486,14 @@ void facets_report(FACETS *facets, BUFFER *wb) { rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv)); k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data); + facets->operations.values.dynamic++; + } + else { + if(!rkv || rkv->empty) + buffer_json_add_array_item_string(wb, FACET_VALUE_UNSET); + else + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); } - else - buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET); } dfe_done(k); buffer_json_array_close(wb); // each row @@ -1458,8 +1550,8 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_member_add_object(wb, "items"); { - buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated); - buffer_json_member_add_uint64(wb, "matched", facets->operations.matched); + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); @@ -1477,7 +1569,36 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends); buffer_json_member_add_uint64(wb, "appends", facets->operations.appends); buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts); + buffer_json_member_add_object(wb, "rows"); + { + buffer_json_member_add_uint64(wb, "created", facets->operations.rows.created); + buffer_json_member_add_uint64(wb, "reused", facets->operations.rows.reused); + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); + } + buffer_json_object_close(wb); // rows + buffer_json_member_add_object(wb, "keys"); + { + buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered); + buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique); + } + buffer_json_object_close(wb); // keys + buffer_json_member_add_object(wb, "values"); + { + buffer_json_member_add_uint64(wb, "registered", facets->operations.values.registered); + buffer_json_member_add_uint64(wb, "transformed", facets->operations.values.transformed); + buffer_json_member_add_uint64(wb, "dynamic", facets->operations.values.dynamic); + buffer_json_member_add_uint64(wb, "empty", facets->operations.values.empty); + buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed); + buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts); + buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts); + } + buffer_json_object_close(wb); // values + buffer_json_member_add_object(wb, "fts"); + { + buffer_json_member_add_uint64(wb, "searches", facets->operations.fts.searches); + } + buffer_json_object_close(wb); // fts } buffer_json_object_close(wb); // items - } diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h index 961be703b4..c34d92d0ef 100644 --- a/libnetdata/facets/facets.h +++ b/libnetdata/facets/facets.h @@ -11,18 +11,20 @@ typedef enum __attribute__((packed)) { } FACETS_ANCHOR_DIRECTION; typedef enum __attribute__((packed)) { - FACET_KEY_OPTION_FACET = (1 << 0), // filterable values - FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value - FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table - FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table - FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS) - FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap - FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter + FACET_KEY_OPTION_FACET = (1 << 0), // filterable values + FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value + FACET_KEY_OPTION_NEVER_FACET = (1 << 2), // never enable this field as facet + FACET_KEY_OPTION_STICKY = (1 << 3), // should be sticky in the table + FACET_KEY_OPTION_VISIBLE = (1 << 4), // should be in the default table + FACET_KEY_OPTION_FTS = (1 << 5), // the key is filterable by full text search (FTS) + FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap + FACET_KEY_OPTION_REORDER = (1 << 7), // give the key a new order id on first encounter } FACET_KEY_OPTIONS; typedef struct facet_row_key_value { const char *tmp; BUFFER *wb; + bool empty; } FACET_ROW_KEY_VALUE; typedef struct facet_row { @@ -64,7 +66,7 @@ void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *v void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut); void facets_add_key_value(FACETS *facets, const char *key, const char *value); -void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len); +void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len); void facets_report(FACETS *facets, BUFFER *wb); void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys); |