Tiering statistics API endpoint (#13420)

* calculator statistics

* added metrics and metrics_pages counters

* implemented API

* updates to match sheet

* updates to match sheet No2

* fix update every calculation for single point pages

* fix lgtm finding

4 files changed, 157 insertions, 5 deletions

diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c
index 20d83e6c37..8b35051d89 100644
--- a/database/engine/rrdengine.c
+++ b/database/engine/rrdengine.c
@@ -9,7 +9,7 @@ rrdeng_stats_t rrdeng_reserved_file_descriptors = 0;
 rrdeng_stats_t global_pg_cache_over_half_dirty_events = 0;
 rrdeng_stats_t global_flushing_pressure_page_deletions = 0;
 
-static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT;
+unsigned rrdeng_pages_per_extent = MAX_PAGES_PER_EXTENT;
 
 #if WORKER_UTILIZATION_MAX_JOB_TYPES < (RRDENG_MAX_OPCODE + 2)
 #error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least (RRDENG_MAX_OPCODE + 2)
@@ -741,7 +741,7 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
          PValue = JudyLFirst(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
          descr = unlikely(NULL == PValue) ? NULL : *PValue ;
 
-         descr != NULL && count != pages_per_extent ;
+         descr != NULL && count != rrdeng_pages_per_extent;
 
          PValue = JudyLNext(pg_cache->committed_page_index.JudyL_array, &Index, PJE0),
          descr = unlikely(NULL == PValue) ? NULL : *PValue) {
@@ -1113,10 +1113,10 @@ static void load_configuration_dynamic(void)
 {
     unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
     if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
-        pages_per_extent = read_num;
+        rrdeng_pages_per_extent = read_num;
     else {
-        error("Invalid dbengine pages per extent %u given. Using %u.", read_num, pages_per_extent);
-        config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", pages_per_extent);
+        error("Invalid dbengine pages per extent %u given. Using %u.", read_num, rrdeng_pages_per_extent);
+        config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", rrdeng_pages_per_extent);
     }
 }
 
diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h
index efb25d2f22..4b383b6226 100644
--- a/database/engine/rrdengine.h
+++ b/database/engine/rrdengine.h
@@ -26,6 +26,8 @@
 
 #endif /* NETDATA_RRD_INTERNALS */
 
+extern unsigned rrdeng_pages_per_extent;
+
 /* Forward declarations */
 struct rrdengine_instance;
 
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index 694055a48e..92e72225e1 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -986,3 +986,100 @@ void rrdeng_prepare_exit(struct rrdengine_instance *ctx)
     //metalog_prepare_exit(ctx->metalog_ctx);
 }
 
+RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
+    RRDENG_SIZE_STATS stats = { 0 };
+
+    for(struct pg_cache_page_index *page_index = ctx->pg_cache.metrics_index.last_page_index;
+        page_index != NULL ;page_index = page_index->prev) {
+        stats.metrics++;
+        stats.metrics_pages += page_index->page_count;
+    }
+
+    for(struct rrdengine_datafile *df = ctx->datafiles.first; df ;df = df->next) {
+        stats.datafiles++;
+
+        for(struct extent_info *ei = df->extents.first; ei ; ei = ei->next) {
+            stats.extents++;
+            stats.extents_compressed_bytes += ei->size;
+
+            for(int p = 0; p < ei->number_of_pages ;p++) {
+                struct rrdeng_page_descr *descr = ei->pages[p];
+
+                usec_t update_every_usec;
+
+                size_t points = descr->page_length / PAGE_POINT_SIZE_BYTES(descr);
+
+                if(likely(points > 1))
+                    update_every_usec = (descr->end_time - descr->start_time) / (points - 1);
+                else {
+                    update_every_usec = default_rrd_update_every * get_tier_grouping(ctx->tier) * USEC_PER_SEC;
+                    stats.single_point_pages++;
+                }
+
+                time_t duration_secs = (time_t)((descr->end_time - descr->start_time + update_every_usec)/USEC_PER_SEC);
+
+                stats.extents_pages++;
+                stats.pages_uncompressed_bytes += descr->page_length;
+                stats.pages_duration_secs += duration_secs;
+                stats.points += points;
+
+                stats.page_types[descr->type].pages++;
+                stats.page_types[descr->type].pages_uncompressed_bytes += descr->page_length;
+                stats.page_types[descr->type].pages_duration_secs += duration_secs;
+                stats.page_types[descr->type].points += points;
+
+                if(!stats.first_t || (descr->start_time - update_every_usec) < stats.first_t)
+                    stats.first_t = (descr->start_time - update_every_usec) / USEC_PER_SEC;
+
+                if(!stats.last_t || descr->end_time > stats.last_t)
+                    stats.last_t = descr->end_time / USEC_PER_SEC;
+            }
+        }
+    }
+
+    internal_error(stats.metrics_pages != stats.extents_pages,
+                   "DBENGINE: metrics pages is %zu, but extents pages is %zu",
+                   stats.metrics_pages, stats.extents_pages);
+
+    stats.database_retention_secs = (time_t)(stats.last_t - stats.first_t);
+
+    if(stats.extents_pages)
+        stats.average_page_size_bytes = (double)stats.pages_uncompressed_bytes / (double)stats.extents_pages;
+
+    if(stats.pages_uncompressed_bytes > 0)
+        stats.average_compression_savings = 100.0 - ((double)stats.extents_compressed_bytes * 100.0 / (double)stats.pages_uncompressed_bytes);
+
+    if(stats.points)
+        stats.average_point_duration_secs = (double)stats.pages_duration_secs / (double)stats.points;
+
+    if(stats.metrics) {
+        stats.average_metric_retention_secs = (double)stats.pages_duration_secs / (double)stats.metrics;
+
+        if(stats.database_retention_secs) {
+            double metric_coverage = stats.average_metric_retention_secs / (double)stats.database_retention_secs;
+            double db_retention_days = (double)stats.database_retention_secs / 86400.0;
+
+            stats.estimated_concurrently_collected_metrics = stats.metrics * metric_coverage;
+
+            stats.ephemeral_metrics_per_day_percent = ((double)stats.metrics * 100.0 / (double)stats.estimated_concurrently_collected_metrics - 100.0) / (double)db_retention_days;
+        }
+    }
+
+    stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index));
+    stats.sizeof_page = struct_natural_alignment(sizeof(struct rrdeng_page_descr));
+    stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
+    stats.sizeof_page_in_cache = struct_natural_alignment(sizeof(struct page_cache_descr));
+    stats.sizeof_point_data = page_type_size[ctx->page_type];
+    stats.sizeof_page_data = RRDENG_BLOCK_SIZE;
+    stats.pages_per_extent = rrdeng_pages_per_extent;
+
+    stats.sizeof_extent = sizeof(struct extent_info);
+    stats.sizeof_page_in_extent = sizeof(struct rrdeng_page_descr *);
+
+    stats.sizeof_metric_in_index = 40;
+    stats.sizeof_page_in_index = 24;
+
+    stats.default_granularity_secs = (size_t)default_rrd_update_every * get_tier_grouping(ctx->tier);
+
+    return stats;
+}
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index ef713eb6eb..e66ec083d5 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -78,4 +78,57 @@ extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
 extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier);
 extern int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
 
+typedef struct rrdengine_size_statistics {
+    size_t default_granularity_secs;
+
+    size_t sizeof_metric;
+    size_t sizeof_metric_in_index;
+    size_t sizeof_page;
+    size_t sizeof_page_in_index;
+    size_t sizeof_extent;
+    size_t sizeof_page_in_extent;
+    size_t sizeof_datafile;
+    size_t sizeof_page_in_cache;
+    size_t sizeof_point_data;
+    size_t sizeof_page_data;
+
+    size_t pages_per_extent;
+
+    size_t datafiles;
+    size_t extents;
+    size_t extents_pages;
+    size_t points;
+    size_t metrics;
+    size_t metrics_pages;
+
+    size_t extents_compressed_bytes;
+    size_t pages_uncompressed_bytes;
+    time_t pages_duration_secs;
+
+    struct {
+        size_t pages;
+        size_t pages_uncompressed_bytes;
+        time_t pages_duration_secs;
+        size_t points;
+    } page_types[256];
+
+    size_t single_point_pages;
+
+    usec_t first_t;
+    usec_t last_t;
+
+    size_t estimated_concurrently_collected_metrics;
+
+    time_t database_retention_secs;
+    double average_compression_savings;
+    double average_point_duration_secs;
+    double average_metric_retention_secs;
+
+    double ephemeral_metrics_per_day_percent;
+
+    double average_page_size_bytes;
+} RRDENG_SIZE_STATS;
+
+extern RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
+
 #endif /* NETDATA_RRDENGINEAPI_H */