diff options
| -rw-r--r-- | database/engine/rrdengine.c | 10 | ||||
| -rw-r--r-- | database/engine/rrdengine.h | 2 | ||||
| -rwxr-xr-x | database/engine/rrdengineapi.c | 97 | ||||
| -rw-r--r-- | database/engine/rrdengineapi.h | 53 | ||||
| -rw-r--r-- | libnetdata/libnetdata.h | 10 | ||||
| -rw-r--r-- | web/api/web_api_v1.c | 97 |
6 files changed, 264 insertions, 5 deletions
diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c index 20d83e6c37..8b35051d89 100644 --- a/database/engine/rrdengine.c +++ b/database/engine/rrdengine.c @@ -9,7 +9,7 @@ rrdeng_stats_t rrdeng_reserved_file_descriptors = 0; rrdeng_stats_t global_pg_cache_over_half_dirty_events = 0; rrdeng_stats_t global_flushing_pressure_page_deletions = 0; -static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT; +unsigned rrdeng_pages_per_extent = MAX_PAGES_PER_EXTENT; #if WORKER_UTILIZATION_MAX_JOB_TYPES < (RRDENG_MAX_OPCODE + 2) #error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least (RRDENG_MAX_OPCODE + 2) @@ -741,7 +741,7 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct PValue = JudyLFirst(pg_cache->committed_page_index.JudyL_array, &Index, PJE0), descr = unlikely(NULL == PValue) ? NULL : *PValue ; - descr != NULL && count != pages_per_extent ; + descr != NULL && count != rrdeng_pages_per_extent; PValue = JudyLNext(pg_cache->committed_page_index.JudyL_array, &Index, PJE0), descr = unlikely(NULL == PValue) ? NULL : *PValue) { @@ -1113,10 +1113,10 @@ static void load_configuration_dynamic(void) { unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT); if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT) - pages_per_extent = read_num; + rrdeng_pages_per_extent = read_num; else { - error("Invalid dbengine pages per extent %u given. Using %u.", read_num, pages_per_extent); - config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", pages_per_extent); + error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent); + config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent); } } diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h index efb25d2f22..4b383b6226 100644 --- a/database/engine/rrdengine.h +++ b/database/engine/rrdengine.h @@ -26,6 +26,8 @@ #endif /* NETDATA_RRD_INTERNALS */ +extern unsigned rrdeng_pages_per_extent; + /* Forward declarations */ struct rrdengine_instance; diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 694055a48e..92e72225e1 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -986,3 +986,100 @@ void rrdeng_prepare_exit(struct rrdengine_instance *ctx) //metalog_prepare_exit(ctx->metalog_ctx); } +RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) { + RRDENG_SIZE_STATS stats = { 0 }; + + for(struct pg_cache_page_index *page_index = ctx->pg_cache.metrics_index.last_page_index; + page_index != NULL ;page_index = page_index->prev) { + stats.metrics++; + stats.metrics_pages += page_index->page_count; + } + + for(struct rrdengine_datafile *df = ctx->datafiles.first; df ;df = df->next) { + stats.datafiles++; + + for(struct extent_info *ei = df->extents.first; ei ; ei = ei->next) { + stats.extents++; + stats.extents_compressed_bytes += ei->size; + + for(int p = 0; p < ei->number_of_pages ;p++) { + struct rrdeng_page_descr *descr = ei->pages[p]; + + usec_t update_every_usec; + + size_t points = descr->page_length / PAGE_POINT_SIZE_BYTES(descr); + + if(likely(points > 1)) + update_every_usec = (descr->end_time - descr->start_time) / (points - 1); + else { + update_every_usec = default_rrd_update_every * get_tier_grouping(ctx->tier) * USEC_PER_SEC; + stats.single_point_pages++; + } + + time_t duration_secs = (time_t)((descr->end_time - descr->start_time + update_every_usec)/USEC_PER_SEC); + + stats.extents_pages++; + stats.pages_uncompressed_bytes += descr->page_length; + stats.pages_duration_secs += duration_secs; + stats.points += points; + + stats.page_types[descr->type].pages++; + stats.page_types[descr->type].pages_uncompressed_bytes += descr->page_length; + stats.page_types[descr->type].pages_duration_secs += duration_secs; + stats.page_types[descr->type].points += points; + + if(!stats.first_t || (descr->start_time - update_every_usec) < stats.first_t) + stats.first_t = (descr->start_time - update_every_usec) / USEC_PER_SEC; + + if(!stats.last_t || descr->end_time > stats.last_t) + stats.last_t = descr->end_time / USEC_PER_SEC; + } + } + } + + internal_error(stats.metrics_pages != stats.extents_pages, + "DBENGINE: metrics pages is %zu, but extents pages is %zu", + stats.metrics_pages, stats.extents_pages); + + stats.database_retention_secs = (time_t)(stats.last_t - stats.first_t); + + if(stats.extents_pages) + stats.average_page_size_bytes = (double)stats.pages_uncompressed_bytes / (double)stats.extents_pages; + + if(stats.pages_uncompressed_bytes > 0) + stats.average_compression_savings = 100.0 - ((double)stats.extents_compressed_bytes * 100.0 / (double)stats.pages_uncompressed_bytes); + + if(stats.points) + stats.average_point_duration_secs = (double)stats.pages_duration_secs / (double)stats.points; + + if(stats.metrics) { + stats.average_metric_retention_secs = (double)stats.pages_duration_secs / (double)stats.metrics; + + if(stats.database_retention_secs) { + double metric_coverage = stats.average_metric_retention_secs / (double)stats.database_retention_secs; + double db_retention_days = (double)stats.database_retention_secs / 86400.0; + + stats.estimated_concurrently_collected_metrics = stats.metrics * metric_coverage; + + stats.ephemeral_metrics_per_day_percent = ((double)stats.metrics * 100.0 / (double)stats.estimated_concurrently_collected_metrics - 100.0) / (double)db_retention_days; + } + } + + stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index)); + stats.sizeof_page = struct_natural_alignment(sizeof(struct rrdeng_page_descr)); + stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile)); + stats.sizeof_page_in_cache = struct_natural_alignment(sizeof(struct page_cache_descr)); + stats.sizeof_point_data = page_type_size[ctx->page_type]; + stats.sizeof_page_data = RRDENG_BLOCK_SIZE; + stats.pages_per_extent = rrdeng_pages_per_extent; + + stats.sizeof_extent = sizeof(struct extent_info); + stats.sizeof_page_in_extent = sizeof(struct rrdeng_page_descr *); + + stats.sizeof_metric_in_index = 40; + stats.sizeof_page_in_index = 24; + + stats.default_granularity_secs = (size_t)default_rrd_update_every * get_tier_grouping(ctx->tier); + + return stats; +} diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h index ef713eb6eb..e66ec083d5 100644 --- a/database/engine/rrdengineapi.h +++ b/database/engine/rrdengineapi.h @@ -78,4 +78,57 @@ extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx); extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier); extern int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t); +typedef struct rrdengine_size_statistics { + size_t default_granularity_secs; + + size_t sizeof_metric; + size_t sizeof_metric_in_index; + size_t sizeof_page; + size_t sizeof_page_in_index; + size_t sizeof_extent; + size_t sizeof_page_in_extent; + size_t sizeof_datafile; + size_t sizeof_page_in_cache; + size_t sizeof_point_data; + size_t sizeof_page_data; + + size_t pages_per_extent; + + size_t datafiles; + size_t extents; + size_t extents_pages; + size_t points; + size_t metrics; + size_t metrics_pages; + + size_t extents_compressed_bytes; + size_t pages_uncompressed_bytes; + time_t pages_duration_secs; + + struct { + size_t pages; + size_t pages_uncompressed_bytes; + time_t pages_duration_secs; + size_t points; + } page_types[256]; + + size_t single_point_pages; + + usec_t first_t; + usec_t last_t; + + size_t estimated_concurrently_collected_metrics; + + time_t database_retention_secs; + double average_compression_savings; + double average_point_duration_secs; + double average_metric_retention_secs; + + double ephemeral_metrics_per_day_percent; + + double average_page_size_bytes; +} RRDENG_SIZE_STATS; + +extern RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx); + #endif /* NETDATA_RRDENGINEAPI_H */ diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index e88e5978a8..8cc6cce9f7 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -360,6 +360,16 @@ extern char *netdata_configured_host_prefix; #define RRD_STORAGE_TIERS 5 +static inline size_t struct_natural_alignment(size_t size) __attribute__((const)); + +#define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) +static inline size_t struct_natural_alignment(size_t size) { + if(unlikely(size % STRUCT_NATURAL_ALIGNMENT)) + size = size + STRUCT_NATURAL_ALIGNMENT - (size % STRUCT_NATURAL_ALIGNMENT); + + return size; +} + # ifdef __cplusplus } # endif diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 6d52b7e821..fc01b95e59 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -1533,6 +1533,100 @@ int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_clie return metric_correlations(host, wb, method, group, group_options, baseline_after, baseline_before, after, before, points, options, timeout); } +#ifndef ENABLE_DBENGINE +int web_client_api_request_v1_dbengine_stats(RRDHOST *host, struct web_client *w, char *url) { + return HTTP_RESP_NOT_FOUND; +} +#else +static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) { + RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]); + + buffer_sprintf(wb, + "\n\t\t\"default_granularity_secs\":%zu" + ",\n\t\t\"sizeof_metric\":%zu" + ",\n\t\t\"sizeof_metric_in_index\":%zu" + ",\n\t\t\"sizeof_page\":%zu" + ",\n\t\t\"sizeof_page_in_index\":%zu" + ",\n\t\t\"sizeof_extent\":%zu" + ",\n\t\t\"sizeof_page_in_extent\":%zu" + ",\n\t\t\"sizeof_datafile\":%zu" + ",\n\t\t\"sizeof_page_in_cache\":%zu" + ",\n\t\t\"sizeof_point_data\":%zu" + ",\n\t\t\"sizeof_page_data\":%zu" + ",\n\t\t\"pages_per_extent\":%zu" + ",\n\t\t\"datafiles\":%zu" + ",\n\t\t\"extents\":%zu" + ",\n\t\t\"extents_pages\":%zu" + ",\n\t\t\"points\":%zu" + ",\n\t\t\"metrics\":%zu" + ",\n\t\t\"metrics_pages\":%zu" + ",\n\t\t\"extents_compressed_bytes\":%zu" + ",\n\t\t\"pages_uncompressed_bytes\":%zu" + ",\n\t\t\"pages_duration_secs\":%ld" + ",\n\t\t\"single_point_pages\":%zu" + ",\n\t\t\"first_t\":%llu" + ",\n\t\t\"last_t\":%llu" + ",\n\t\t\"database_retention_secs\":%ld" + ",\n\t\t\"average_compression_savings\":%0.2f" + ",\n\t\t\"average_point_duration_secs\":%0.2f" + ",\n\t\t\"average_metric_retention_secs\":%0.2f" + ",\n\t\t\"ephemeral_metrics_per_day_percent\":%0.2f" + ",\n\t\t\"average_page_size_bytes\":%0.2f" + ",\n\t\t\"estimated_concurrently_collected_metrics\":%zu" + , stats.default_granularity_secs + , stats.sizeof_metric + , stats.sizeof_metric_in_index + , stats.sizeof_page + , stats.sizeof_page_in_index + , stats.sizeof_extent + , stats.sizeof_page_in_extent + , stats.sizeof_datafile + , stats.sizeof_page_in_cache + , stats.sizeof_point_data + , stats.sizeof_page_data + , stats.pages_per_extent + , stats.datafiles + , stats.extents + , stats.extents_pages + , stats.points + , stats.metrics + , stats.metrics_pages + , stats.extents_compressed_bytes + , stats.pages_uncompressed_bytes + , stats.pages_duration_secs + , stats.single_point_pages + , stats.first_t + , stats.last_t + , stats.database_retention_secs + , stats.average_compression_savings + , stats.average_point_duration_secs + , stats.average_metric_retention_secs + , stats.ephemeral_metrics_per_day_percent + , stats.average_page_size_bytes + , stats.estimated_concurrently_collected_metrics + ); +} +int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) { + if (!netdata_ready) + return HTTP_RESP_BACKEND_FETCH_FAILED; + + BUFFER *wb = w->response.data; + buffer_flush(wb); + wb->contenttype = CT_APPLICATION_JSON; + buffer_no_cacheable(wb); + + buffer_strcat(wb, "{"); + for(int tier = 0; tier < storage_tiers ;tier++) { + buffer_sprintf(wb, "%s\n\t\"tier%d\": {", tier?",":"", tier); + web_client_api_v1_dbengine_stats_for_tier(wb, tier); + buffer_strcat(wb, "\n\t}"); + } + buffer_strcat(wb, "\n}"); + + return HTTP_RESP_OK; +} +#endif + static struct api_command { const char *command; uint32_t hash; @@ -1569,6 +1663,9 @@ static struct api_command { { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health }, { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state }, { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations }, + + { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_dbengine_stats }, + // terminator { NULL, 0, WEB_CLIENT_ACL_NONE, NULL }, }; |