From cad1692853791d172b0c683a3105803aa62db114 Mon Sep 17 00:00:00 2001 From: vkalintiris Date: Tue, 19 Mar 2024 14:27:03 +0200 Subject: Add constant page type --- src/daemon/global_statistics.c | 53 ++++++++++++++++ src/daemon/global_statistics.h | 4 ++ src/daemon/main.c | 3 + src/daemon/static_threads.c | 2 +- src/database/engine/page.c | 111 +++++++++++++++++++++++++++++++--- src/database/engine/page_test.cc | 10 +-- src/database/engine/pdc.c | 17 +++++- src/database/engine/rrddiskprotocol.h | 12 +++- src/database/engine/rrdengine.c | 6 +- src/database/engine/rrdengineapi.c | 14 +++-- 10 files changed, 203 insertions(+), 29 deletions(-) diff --git a/src/daemon/global_statistics.c b/src/daemon/global_statistics.c index ffc90c54a6..e9200a1366 100644 --- a/src/daemon/global_statistics.c +++ b/src/daemon/global_statistics.c @@ -70,6 +70,9 @@ static struct global_statistics { uint64_t tier0_disk_compressed_bytes; uint64_t tier0_disk_uncompressed_bytes; + uint64_t tier0_raw_pages; + uint64_t tier0_constant_pages; + uint64_t db_points_stored_per_tier[RRD_STORAGE_TIERS]; } global_statistics = { @@ -89,6 +92,9 @@ static struct global_statistics { .tier0_hot_gorilla_buffers = 0, .tier0_disk_compressed_bytes = 0, .tier0_disk_uncompressed_bytes = 0, + + .tier0_raw_pages = 0, + .tier0_constant_pages = 0, }; void global_statistics_rrdset_done_chart_collection_completed(size_t *points_read_per_tier_array) { @@ -129,6 +135,18 @@ void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size) { __atomic_fetch_add(&global_statistics.tier0_disk_uncompressed_bytes, size, __ATOMIC_RELAXED); } +void global_statistic_raw_page_new() { + __atomic_fetch_add(&global_statistics.tier0_raw_pages, 1, __ATOMIC_RELAXED); +} + +void global_statistic_constant_page_new() { + __atomic_fetch_add(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED); +} + +void global_statistic_constant_page_rm() { + __atomic_fetch_sub(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED); +} + void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) { switch(query_source) { case QUERY_SOURCE_API_DATA: @@ -236,6 +254,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t gs->tier0_disk_compressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_compressed_bytes, __ATOMIC_RELAXED); gs->tier0_disk_uncompressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_uncompressed_bytes, __ATOMIC_RELAXED); + gs->tier0_raw_pages = __atomic_load_n(&global_statistics.tier0_raw_pages, __ATOMIC_RELAXED); + gs->tier0_constant_pages = __atomic_load_n(&global_statistics.tier0_constant_pages, __ATOMIC_RELAXED); + for(size_t tier = 0; tier < storage_tiers ;tier++) gs->db_points_stored_per_tier[tier] = __atomic_load_n(&global_statistics.db_points_stored_per_tier[tier], __ATOMIC_RELAXED); @@ -920,6 +941,38 @@ static void global_statistics_charts(void) { rrdset_done(st_tier0_compression_info); } #endif + + { + static RRDSET *st_t0_page_type = NULL; + + static RRDDIM *rd_raw_pages = NULL; + static RRDDIM *rd_constant_pages = NULL; + + if (unlikely(!st_t0_page_type)) { + st_t0_page_type = rrdset_create_localhost( + "netdata" + , "tier0_page_type" + , NULL + , "tier0_page_type" + , NULL + , "Tier 0 page type" + , "pages" + , "netdata" + , "stats" + , 131006 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_raw_pages = rrddim_add(st_t0_page_type, "raw", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_constant_pages = rrddim_add(st_t0_page_type, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + rrddim_set_by_pointer(st_t0_page_type, rd_raw_pages, (collected_number) gs.tier0_raw_pages); + rrddim_set_by_pointer(st_t0_page_type, rd_constant_pages, (collected_number) gs.tier0_constant_pages); + + rrdset_done(st_t0_page_type); + } } // ---------------------------------------------------------------------------- diff --git a/src/daemon/global_statistics.h b/src/daemon/global_statistics.h index 44717c6cf4..88df1c6519 100644 --- a/src/daemon/global_statistics.h +++ b/src/daemon/global_statistics.h @@ -50,6 +50,10 @@ void global_statistics_gorilla_buffer_add_hot(); void global_statistics_tier0_disk_compressed_bytes(uint32_t size); void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size); +void global_statistic_raw_page_new(); +void global_statistic_constant_page_new(); +void global_statistic_constant_page_rm(); + void global_statistics_web_request_completed(uint64_t dt, uint64_t bytes_received, uint64_t bytes_sent, diff --git a/src/daemon/main.c b/src/daemon/main.c index de22a90f43..e7c69d3927 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -1172,6 +1172,9 @@ static void get_netdata_configured_variables() { const char *page_type = config_get(CONFIG_SECTION_DB, "dbengine page type", "raw"); if (strcmp(page_type, "gorilla") == 0) { tier_page_type[0] = PAGE_GORILLA_METRICS; + } + else if (strcmp(page_type, "constant") == 0) { + tier_page_type[0] = PAGE_CONSTANT_METRICS; } else if (strcmp(page_type, "raw") != 0) { netdata_log_error("Invalid dbengine page type ''%s' given. Defaulting to 'raw'.", page_type); } diff --git a/src/daemon/static_threads.c b/src/daemon/static_threads.c index e03819761e..ce0fd1ad02 100644 --- a/src/daemon/static_threads.c +++ b/src/daemon/static_threads.c @@ -61,7 +61,7 @@ const struct netdata_static_thread static_threads_common[] = { .config_name = "netdata monitoring", .env_name = "NETDATA_INTERNALS_MONITORING", .global_variable = &global_statistics_enabled, - .enabled = 0, + .enabled = 1, .thread = NULL, .init_routine = NULL, .start_routine = global_statistics_main diff --git a/src/database/engine/page.c b/src/database/engine/page.c index be4a02d3bc..0b737921ce 100644 --- a/src/database/engine/page.c +++ b/src/database/engine/page.c @@ -2,6 +2,7 @@ #include "page.h" +#include "daemon/global_statistics.h" #include "libnetdata/libnetdata.h" typedef enum __attribute__((packed)) { @@ -20,6 +21,10 @@ typedef struct { uint32_t size; } page_raw_t; +typedef struct { + storage_number v; + uint32_t n; +} page_constant_t; typedef struct { size_t num_buffers; @@ -45,6 +50,7 @@ struct pgd { union { page_raw_t raw; page_gorilla_t gorilla; + page_constant_t constant; }; }; @@ -165,7 +171,9 @@ PGD *pgd_create(uint8_t type, uint32_t slots) pg->states = PGD_STATE_CREATED_FROM_COLLECTOR; switch (type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: + global_statistic_raw_page_new(); + // intentional fall-through case PAGE_TIER: { uint32_t size = slots * page_type_size[type]; @@ -176,6 +184,16 @@ PGD *pgd_create(uint8_t type, uint32_t slots) pg->raw.data = pgd_data_aral_alloc(size); break; } + case PAGE_CONSTANT_METRICS: { + global_statistic_constant_page_new(); + + internal_fatal(slots == 1, + "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type); + + pg->constant.v = SN_EMPTY_SLOT; + pg->constant.n = 0; + break; + } case PAGE_GORILLA_METRICS: { internal_fatal(slots == 1, "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type); @@ -222,7 +240,7 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) switch (type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: pg->raw.size = size; pg->used = size / page_type_size[type]; @@ -231,6 +249,12 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size) pg->raw.data = pgd_data_aral_alloc(size); memcpy(pg->raw.data, base, size); break; + case PAGE_CONSTANT_METRICS: { + memcpy(&pg->constant.v, base, sizeof(storage_number)); + memcpy(&pg->constant.n, base + sizeof(storage_number), sizeof(uint32_t)); + pg->used = pg->slots = pg->constant.n; + break; + } case PAGE_GORILLA_METRICS: internal_fatal(size == 0, "Asked to create page with 0 data!!!"); internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size"); @@ -268,10 +292,12 @@ void pgd_free(PGD *pg) switch (pg->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: pgd_data_aral_free(pg->raw.data, pg->raw.size); break; + case PAGE_CONSTANT_METRICS: + break; case PAGE_GORILLA_METRICS: { if (pg->states & PGD_STATE_CREATED_FROM_DISK) { @@ -365,10 +391,13 @@ uint32_t pgd_memory_footprint(PGD *pg) size_t footprint = 0; switch (pg->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: footprint = sizeof(PGD) + pg->raw.size; break; + case PAGE_CONSTANT_METRICS: + footprint = sizeof(PGD); + break; case PAGE_GORILLA_METRICS: { if (pg->states & PGD_STATE_CREATED_FROM_DISK) footprint = sizeof(PGD) + pg->raw.size; @@ -393,7 +422,7 @@ uint32_t pgd_disk_footprint(PGD *pg) size_t size = 0; switch (pg->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: { uint32_t used_size = pg->used * page_type_size[pg->type]; internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size"); @@ -401,6 +430,10 @@ uint32_t pgd_disk_footprint(PGD *pg) break; } + case PAGE_CONSTANT_METRICS: { + size = sizeof(storage_number) + sizeof(uint32_t); + break; + } case PAGE_GORILLA_METRICS: { if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR || pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING || @@ -443,11 +476,21 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size) pgd_disk_footprint(pg), dst_size); switch (pg->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: memcpy(dst, pg->raw.data, dst_size); break; - case PAGE_GORILLA_METRICS: { + case PAGE_CONSTANT_METRICS: + { + memcpy(dst, &pg->constant.v, sizeof(storage_number)); + + pg->constant.n = pg->used; + memcpy(dst + sizeof(storage_number), &pg->constant.n, sizeof(uint32_t)); + + break; + } + case PAGE_GORILLA_METRICS: + { if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0) fatal("Copying to extent is supported only for PGDs that are scheduled for flushing."); @@ -500,7 +543,7 @@ void pgd_append_point(PGD *pg, fatal("Data collection on page already scheduled for flushing"); switch (pg->type) { - case PAGE_METRICS: { + case PAGE_RAW_METRICS: { storage_number *tier0_metric_data = (storage_number *)pg->raw.data; storage_number t = pack_storage_number(n, flags); tier0_metric_data[pg->used++] = t; @@ -510,6 +553,43 @@ void pgd_append_point(PGD *pg, break; } + case PAGE_CONSTANT_METRICS: { + storage_number t = pack_storage_number(n, flags); + + if (pg->used == 0) + { + pg->constant.v = t; + pg->used = 1; + } + else if (pg->constant.v == t) + { + pg->used++; + } + else + { + storage_number v = pg->constant.v; + + global_statistic_constant_page_rm(); + global_statistic_raw_page_new(); + + pg->type = PAGE_RAW_METRICS; + uint32_t size = pg->slots * page_type_size[pg->type]; + pg->raw.size = size; + pg->raw.data = pgd_data_aral_alloc(size); + + storage_number *data = (storage_number *) pg->raw.data; + for (uint32_t i = 0; i != pg->used; i++) + data[i] = v; + + data[pg->used++] = t; + break; + } + + if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t)) + pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY; + + break; + } case PAGE_TIER: { storage_number_tier1_t *tier12_metric_data = (storage_number_tier1_t *)pg->raw.data; storage_number_tier1_t t; @@ -560,8 +640,9 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position) PGD *pg = pgdc->pgd; switch (pg->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: + case PAGE_CONSTANT_METRICS: pgdc->slots = pgdc->pgd->used; break; case PAGE_GORILLA_METRICS: { @@ -634,7 +715,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, switch (pgdc->pgd->type) { - case PAGE_METRICS: { + case PAGE_RAW_METRICS: { storage_number *array = (storage_number *) pgdc->pgd->raw.data; storage_number n = array[pgdc->position++]; @@ -645,6 +726,16 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused, return true; } + case PAGE_CONSTANT_METRICS: { + storage_number n = pgdc->pgd->constant.v; + + sp->min = sp->max = sp->sum = unpack_storage_number(n); + sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS); + sp->count = 1; + sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0; + + return true; + } case PAGE_TIER: { storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data; storage_number_tier1_t n = array[pgdc->position++]; diff --git a/src/database/engine/page_test.cc b/src/database/engine/page_test.cc index d61299bc48..6730fbada6 100644 --- a/src/database/engine/page_test.cc +++ b/src/database/engine/page_test.cc @@ -38,7 +38,7 @@ static uint8_t page_type = PAGE_GORILLA_METRICS; static size_t slots_for_page(size_t n) { switch (page_type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: return 1024; case PAGE_GORILLA_METRICS: return n; @@ -200,7 +200,7 @@ TEST(PGD, MemoryFootprint) { uint32_t footprint = 0; switch (pgd_type(pg)) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: footprint = slots * sizeof(uint32_t); break; case PAGE_GORILLA_METRICS: @@ -225,7 +225,7 @@ TEST(PGD, MemoryFootprint) { uint32_t abs_error = 0; switch (pgd_type(pg)) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: abs_error = 128; break; case PAGE_GORILLA_METRICS: @@ -256,7 +256,7 @@ TEST(PGD, DiskFootprint) { uint32_t footprint = 0; switch (pgd_type(pg)) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: footprint = used_slots * sizeof(uint32_t); break; case PAGE_GORILLA_METRICS: @@ -279,7 +279,7 @@ TEST(PGD, DiskFootprint) { } switch (pgd_type(pg)) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: footprint = used_slots * sizeof(uint32_t); break; case PAGE_GORILLA_METRICS: diff --git a/src/database/engine/pdc.c b/src/database/engine/pdc.c index 1b57a49ba6..4808baefa4 100644 --- a/src/database/engine/pdc.c +++ b/src/database/engine/pdc.c @@ -635,7 +635,7 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_ size_t entries = 0; switch (descr->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: end_time_s = descr->end_time_ut / USEC_PER_SEC; entries = 0; @@ -644,6 +644,10 @@ inline VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_ end_time_s = start_time_s + descr->gorilla.delta_time_s; entries = descr->gorilla.entries; break; + case PAGE_CONSTANT_METRICS: + end_time_s = start_time_s + descr->constant.delta_time_s; + entries = descr->constant.entries; + break; default: // Nothing to do. Validate page will notify the user. break; @@ -689,7 +693,7 @@ VALIDATED_PAGE_DESCRIPTOR validate_page( bool known_page_type = true; switch (page_type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: // always calculate entries by size vd.entries = page_entries_by_size(vd.page_length, vd.point_size); @@ -702,6 +706,10 @@ VALIDATED_PAGE_DESCRIPTOR validate_page( internal_fatal(entries == 0, "0 number of entries found on gorilla page"); vd.entries = entries; break; + case PAGE_CONSTANT_METRICS: + internal_fatal(entries == 0, "0 number of entries found on constant page"); + vd.entries = entries; + break; default: known_page_type = false; break; @@ -873,13 +881,16 @@ static void epdl_extent_loading_error_log(struct rrdengine_instance *ctx, EPDL * if (descr) { start_time_s = (time_t)(descr->start_time_ut / USEC_PER_SEC); switch (descr->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: end_time_s = (time_t)(descr->end_time_ut / USEC_PER_SEC); break; case PAGE_GORILLA_METRICS: end_time_s = (time_t) start_time_s + (descr->gorilla.delta_time_s); break; + case PAGE_CONSTANT_METRICS: + end_time_s = (time_t) start_time_s + (descr->constant.delta_time_s); + break; } uuid_unparse_lower(descr->uuid, uuid); used_descr = true; diff --git a/src/database/engine/rrddiskprotocol.h b/src/database/engine/rrddiskprotocol.h index 86b41f0b3e..353e9fef64 100644 --- a/src/database/engine/rrddiskprotocol.h +++ b/src/database/engine/rrddiskprotocol.h @@ -36,10 +36,11 @@ struct rrdeng_df_sb { /* * Page types */ -#define PAGE_METRICS (0) +#define PAGE_RAW_METRICS (0) #define PAGE_TIER (1) #define PAGE_GORILLA_METRICS (2) -#define PAGE_TYPE_MAX 2 // Maximum page type (inclusive) +#define PAGE_CONSTANT_METRICS (3) +#define PAGE_TYPE_MAX 3 // Maximum page type (inclusive) /* * Data file page descriptor @@ -56,6 +57,11 @@ struct rrdeng_extent_page_descr { uint32_t delta_time_s; } gorilla __attribute__((packed)); + struct { + uint32_t entries; + uint32_t delta_time_s; + } constant __attribute__((packed)); + uint64_t end_time_ut; }; } __attribute__ ((packed)); @@ -127,4 +133,4 @@ struct rrdeng_jf_store_data { struct rrdeng_extent_page_descr descr[]; } __attribute__ ((packed)); -#endif /* NETDATA_RRDDISKPROTOCOL_H */ \ No newline at end of file +#endif /* NETDATA_RRDDISKPROTOCOL_H */ diff --git a/src/database/engine/rrdengine.c b/src/database/engine/rrdengine.c index bae0fc4650..b21baacd2d 100644 --- a/src/database/engine/rrdengine.c +++ b/src/database/engine/rrdengine.c @@ -844,7 +844,7 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta header->descr[i].start_time_ut = descr->start_time_ut; switch (descr->type) { - case PAGE_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: header->descr[i].end_time_ut = descr->end_time_ut; break; @@ -852,6 +852,10 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta header->descr[i].gorilla.delta_time_s = (uint32_t) ((descr->end_time_ut - descr->start_time_ut) / USEC_PER_SEC); header->descr[i].gorilla.entries = pgd_slots_used(descr->pgd); break; + case PAGE_CONSTANT_METRICS: + header->descr[i].constant.delta_time_s = (uint32_t) ((descr->end_time_ut - descr->start_time_ut) / USEC_PER_SEC); + header->descr[i].constant.entries = pgd_slots_used(descr->pgd); + break; default: fatal("Unknown page type: %uc", descr->type); } diff --git a/src/database/engine/rrdengineapi.c b/src/database/engine/rrdengineapi.c index b9c2b59801..e93affdedf 100755 --- a/src/database/engine/rrdengineapi.c +++ b/src/database/engine/rrdengineapi.c @@ -16,7 +16,7 @@ struct rrdengine_instance multidb_ctx_storage_tier4; #error RRD_STORAGE_TIERS is not 5 - you need to add allocations here #endif struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS]; -uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER}; +uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_CONSTANT_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER}; #if defined(ENV32BIT) size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192}; @@ -24,14 +24,15 @@ size_t tier_page_size[RRD_STORAGE_TIERS] = {2048, 1024, 192, 192, 192}; size_t tier_page_size[RRD_STORAGE_TIERS] = {4096, 2048, 384, 384, 384}; #endif -#if PAGE_TYPE_MAX != 2 -#error PAGE_TYPE_MAX is not 2 - you need to add allocations here +#if PAGE_TYPE_MAX != 3 +#error PAGE_TYPE_MAX is not 3 - you need to add allocations here #endif size_t page_type_size[256] = { - [PAGE_METRICS] = sizeof(storage_number), + [PAGE_RAW_METRICS] = sizeof(storage_number), [PAGE_TIER] = sizeof(storage_number_tier1_t), - [PAGE_GORILLA_METRICS] = sizeof(storage_number) + [PAGE_GORILLA_METRICS] = sizeof(storage_number), + [PAGE_CONSTANT_METRICS] = sizeof(storage_number), }; __attribute__((constructor)) void initialize_multidb_ctx(void) { @@ -457,7 +458,8 @@ static PGD *rrdeng_alloc_new_page_data(struct rrdeng_collect_handle *handle, siz *data_size = size; switch (ctx->config.page_type) { - case PAGE_METRICS: + case PAGE_CONSTANT_METRICS: + case PAGE_RAW_METRICS: case PAGE_TIER: d = pgd_create(ctx->config.page_type, slots); break; -- cgit v1.2.3