DBENGINE v2 - improvements part 1 (#14251)

* allow running multiple evictors and flushers

* flipped aggressive and critical evictions

* dont run more than 1 evictor

* switch to batch evictions when the size of the cache is critical

* remove batching of evictions

* dedup extent load pending requests

* accounting for merged extents

* always use double linked list

* add extent merging to the overall cache hit ratio

* support requeuing merged extents to higher priorities

* fix function name

* query planner now prefers higher tiers even when they miss some data at the end, which it fills from lower tiers; adding the option "plan" to jsonwrap now renders the query plan

* update statistics after every dimension completes

* use the retention of all tiers to calculate coverage per tier

* use the original window of the query for the planner

* give 2.5% befenit for each higher tier

* update cmd->priority so that it be requeued multiple times

* merged extent pages is a cache hit

* fixed dbegnine cache hit stats

3 files changed, 43 insertions, 55 deletions

diff --git a/daemon/event_loop.c b/daemon/event_loop.c
index 13d3a5822c..86a0220657 100644
--- a/daemon/event_loop.c
+++ b/daemon/event_loop.c
@@ -18,7 +18,6 @@ void register_libuv_worker_jobs() {
     worker_register_job_name(UV_EVENT_COMMIT_PAGE_CB, "commit cb");
     worker_register_job_name(UV_EVENT_FLUSH_PAGES_CB, "flush cb");
     worker_register_job_name(UV_EVENT_PAGE_LOOKUP, "page lookup");
-    worker_register_job_name(UV_EVENT_METRIC_LOOKUP, "metric lookup");
     worker_register_job_name(UV_EVENT_PAGE_POPULATION, "populate page");
     worker_register_job_name(UV_EVENT_EXT_DECOMPRESSION, "extent decompression");
     worker_register_job_name(UV_EVENT_READ_MMAP_EXTENT, "read extent (mmap)");
diff --git a/daemon/event_loop.h b/daemon/event_loop.h
index e332c253cd..010daea513 100644
--- a/daemon/event_loop.h
+++ b/daemon/event_loop.h
@@ -11,7 +11,6 @@ enum event_loop_job {
     UV_EVENT_FLUSH_PAGES_CB,
     UV_EVENT_EXT_DECOMPRESSION,
     UV_EVENT_PAGE_LOOKUP,
-    UV_EVENT_METRIC_LOOKUP,
     UV_EVENT_PAGE_POPULATION,
     UV_EVENT_READ_MMAP_EXTENT,
     UV_EVENT_EXTENT_PROCESSING,
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 3bd3822cca..40e8795b82 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -1373,8 +1373,8 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
                     localhost->rrd_update_every,
                     RRDSET_TYPE_AREA);
 
-            ptrs->rd_pgc_memory_evictions_aggressive = rrddim_add(ptrs->st_pgc_memory_events, "evictions critical", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
-            ptrs->rd_pgc_memory_evictions_critical   = rrddim_add(ptrs->st_pgc_memory_events, "evictions aggressive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            ptrs->rd_pgc_memory_evictions_aggressive = rrddim_add(ptrs->st_pgc_memory_events, "evictions aggressive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            ptrs->rd_pgc_memory_evictions_critical   = rrddim_add(ptrs->st_pgc_memory_events, "evictions critical", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
             ptrs->rd_pgc_memory_flushes_critical     = rrddim_add(ptrs->st_pgc_memory_events, "flushes critical", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
 
             buffer_free(id);
@@ -1671,11 +1671,9 @@ static void dbengine2_statistics_charts(void) {
     {
         static RRDSET *st_cache_hit_ratio = NULL;
         static RRDDIM *rd_hit_ratio = NULL;
-        static RRDDIM *rd_preloaded_ratio = NULL;
-        static RRDDIM *rd_extent_ratio = NULL;
-        static RRDDIM *rd_parallel_load_ratio = NULL;
-        static RRDDIM *rd_before_allocation_ratio = NULL;
-        static RRDDIM *rd_insert_conflict_ratio = NULL;
+        static RRDDIM *rd_main_cache_hit_ratio = NULL;
+        static RRDDIM *rd_extent_cache_hit_ratio = NULL;
+        static RRDDIM *rd_extent_merge_hit_ratio = NULL;
 
         if (unlikely(!st_cache_hit_ratio)) {
             st_cache_hit_ratio = rrdset_create_localhost(
@@ -1693,60 +1691,55 @@ static void dbengine2_statistics_charts(void) {
                     RRDSET_TYPE_LINE);
 
             rd_hit_ratio = rrddim_add(st_cache_hit_ratio, "overall", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
-            rd_preloaded_ratio = rrddim_add(st_cache_hit_ratio, "main cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
-            rd_extent_ratio = rrddim_add(st_cache_hit_ratio, "extent cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
-            rd_parallel_load_ratio = rrddim_add(st_cache_hit_ratio, "parallel load", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
-            rd_before_allocation_ratio = rrddim_add(st_cache_hit_ratio, "before allocation", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
-            rd_insert_conflict_ratio = rrddim_add(st_cache_hit_ratio, "insert conflict", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
+            rd_main_cache_hit_ratio = rrddim_add(st_cache_hit_ratio, "main cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
+            rd_extent_cache_hit_ratio = rrddim_add(st_cache_hit_ratio, "extent cache", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
+            rd_extent_merge_hit_ratio = rrddim_add(st_cache_hit_ratio, "extent merge", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
         }
         priority++;
 
-        size_t pages_to_load = cache_efficiency_stats.pages_to_load_from_disk - cache_efficiency_stats_old.pages_to_load_from_disk;
-        size_t pages_hit_parallel_load = cache_efficiency_stats.pages_load_ok_preloaded - cache_efficiency_stats_old.pages_load_ok_preloaded;
-        size_t pages_hit_before_allocation = cache_efficiency_stats.pages_load_ok_loaded_but_cache_hit_before_allocation - cache_efficiency_stats_old.pages_load_ok_loaded_but_cache_hit_before_allocation;
-        size_t pages_hit_insert_conflict = cache_efficiency_stats.pages_load_ok_loaded_but_cache_hit_while_inserting - cache_efficiency_stats_old.pages_load_ok_loaded_but_cache_hit_while_inserting;
-        size_t pages_hit_cached_extent = cache_efficiency_stats.pages_data_source_extent_cache - cache_efficiency_stats_old.pages_data_source_extent_cache;
-        size_t pages_hit_at_query_plan = cache_efficiency_stats.pages_meta_source_main_cache - cache_efficiency_stats_old.pages_meta_source_main_cache;
+        size_t delta_pages_total = cache_efficiency_stats.pages_total - cache_efficiency_stats_old.pages_total;
+        size_t delta_pages_to_load_from_disk = cache_efficiency_stats.pages_to_load_from_disk - cache_efficiency_stats_old.pages_to_load_from_disk;
+        size_t delta_extents_loaded_from_disk = cache_efficiency_stats.extents_loaded_from_disk - cache_efficiency_stats_old.extents_loaded_from_disk;
 
-        size_t pages_total_hit = pages_hit_at_query_plan + pages_hit_before_allocation + pages_hit_insert_conflict + pages_hit_cached_extent;
-        size_t pages_total = cache_efficiency_stats.pages_total - cache_efficiency_stats_old.pages_total;
+        size_t delta_pages_data_source_main_cache = cache_efficiency_stats.pages_data_source_main_cache - cache_efficiency_stats_old.pages_data_source_main_cache;
+        size_t delta_pages_pending_found_in_cache_at_pass4 = cache_efficiency_stats.pages_data_source_main_cache_at_pass4 - cache_efficiency_stats_old.pages_data_source_main_cache_at_pass4;
+
+        size_t delta_pages_data_source_extent_cache = cache_efficiency_stats.pages_data_source_extent_cache - cache_efficiency_stats_old.pages_data_source_extent_cache;
+        size_t delta_pages_load_extent_merged = cache_efficiency_stats.pages_load_extent_merged - cache_efficiency_stats_old.pages_load_extent_merged;
+
+        size_t pages_total_hit = delta_pages_total - delta_extents_loaded_from_disk;
 
         static size_t overall_hit_ratio = 100;
-        size_t preloaded_hit_ratio = 0, extent_hit_ratio = 0;
-        size_t before_allocation_hit_ratio = 0, insert_conflict_hit_ratio = 0, parallel_load_hit_ratio = 0;
-        if(pages_total) {
-            if(pages_total_hit > pages_total)
-                pages_total_hit = pages_total;
+        size_t main_cache_hit_ratio = 0, extent_cache_hit_ratio = 0, extent_merge_hit_ratio = 0;
+        if(delta_pages_total) {
+            if(pages_total_hit > delta_pages_total)
+                pages_total_hit = delta_pages_total;
+
+            overall_hit_ratio = pages_total_hit * 100 * 10000 / delta_pages_total;
 
-            overall_hit_ratio = pages_total_hit * 100 * 10000 / pages_total;
+            size_t delta_pages_main_cache = delta_pages_data_source_main_cache + delta_pages_pending_found_in_cache_at_pass4;
+            if(delta_pages_main_cache > delta_pages_total)
+                delta_pages_main_cache = delta_pages_total;
 
-            preloaded_hit_ratio = pages_hit_at_query_plan * 100 * 10000 / pages_total;
+            main_cache_hit_ratio = delta_pages_main_cache * 100 * 10000 / delta_pages_total;
         }
 
-        if(pages_to_load) {
-            extent_hit_ratio = pages_hit_cached_extent * 100 * 10000 / pages_to_load;
-            if(extent_hit_ratio > 100 * 10000)
-                extent_hit_ratio = 100 * 10000;
+        if(delta_pages_to_load_from_disk) {
+            if(delta_pages_data_source_extent_cache > delta_pages_to_load_from_disk)
+                delta_pages_data_source_extent_cache = delta_pages_to_load_from_disk;
 
-            parallel_load_hit_ratio = pages_hit_parallel_load * 100 * 10000 / pages_to_load;
-            if(parallel_load_hit_ratio > 100 * 10000)
-                parallel_load_hit_ratio = 100 * 10000;
+            extent_cache_hit_ratio = delta_pages_data_source_extent_cache * 100 * 10000 / delta_pages_to_load_from_disk;
 
-            before_allocation_hit_ratio = pages_hit_before_allocation * 100 * 10000 / pages_to_load;
-            if(before_allocation_hit_ratio > 100 * 10000)
-                before_allocation_hit_ratio = 100 * 10000;
+            if(delta_pages_load_extent_merged > delta_pages_to_load_from_disk)
+                delta_pages_load_extent_merged = delta_pages_to_load_from_disk;
 
-            insert_conflict_hit_ratio = pages_hit_insert_conflict * 100 * 10000 / pages_to_load;
-            if(insert_conflict_hit_ratio > 100 * 10000)
-                insert_conflict_hit_ratio = 100 * 10000;
+            extent_merge_hit_ratio = delta_pages_load_extent_merged * 100 * 10000 / delta_pages_to_load_from_disk;
         }
 
         rrddim_set_by_pointer(st_cache_hit_ratio, rd_hit_ratio, (collected_number)overall_hit_ratio);
-        rrddim_set_by_pointer(st_cache_hit_ratio, rd_preloaded_ratio, (collected_number)preloaded_hit_ratio);
-        rrddim_set_by_pointer(st_cache_hit_ratio, rd_extent_ratio, (collected_number)extent_hit_ratio);
-        rrddim_set_by_pointer(st_cache_hit_ratio, rd_parallel_load_ratio, (collected_number)parallel_load_hit_ratio);
-        rrddim_set_by_pointer(st_cache_hit_ratio, rd_before_allocation_ratio, (collected_number)before_allocation_hit_ratio);
-        rrddim_set_by_pointer(st_cache_hit_ratio, rd_insert_conflict_ratio, (collected_number)insert_conflict_hit_ratio);
+        rrddim_set_by_pointer(st_cache_hit_ratio, rd_main_cache_hit_ratio, (collected_number)main_cache_hit_ratio);
+        rrddim_set_by_pointer(st_cache_hit_ratio, rd_extent_cache_hit_ratio, (collected_number)extent_cache_hit_ratio);
+        rrddim_set_by_pointer(st_cache_hit_ratio, rd_extent_merge_hit_ratio, (collected_number)extent_merge_hit_ratio);
 
         rrdset_done(st_cache_hit_ratio);
     }
@@ -1880,7 +1873,7 @@ static void dbengine2_statistics_charts(void) {
         }
         priority++;
 
-        rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_main_cache, (collected_number)cache_efficiency_stats.pages_data_source_main_cache);
+        rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_main_cache, (collected_number)cache_efficiency_stats.pages_data_source_main_cache + (collected_number)cache_efficiency_stats.pages_data_source_main_cache_at_pass4);
         rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_disk, (collected_number)cache_efficiency_stats.pages_to_load_from_disk);
         rrddim_set_by_pointer(st_query_pages_data_source, rd_pages_extent_cache, (collected_number)cache_efficiency_stats.pages_data_source_extent_cache);
 
@@ -1918,7 +1911,7 @@ static void dbengine2_statistics_charts(void) {
         }
         priority++;
 
-        rrddim_set_by_pointer(st_query_next_page, rd_pass4, (collected_number)cache_efficiency_stats.pages_pending_found_in_cache_at_pass4);
+        rrddim_set_by_pointer(st_query_next_page, rd_pass4, (collected_number)cache_efficiency_stats.pages_data_source_main_cache_at_pass4);
         rrddim_set_by_pointer(st_query_next_page, rd_wait_failed, (collected_number)cache_efficiency_stats.page_next_wait_failed);
         rrddim_set_by_pointer(st_query_next_page, rd_nowait_failed, (collected_number)cache_efficiency_stats.page_next_nowait_failed);
         rrddim_set_by_pointer(st_query_next_page, rd_wait_loaded, (collected_number)cache_efficiency_stats.page_next_wait_loaded);
@@ -1977,11 +1970,10 @@ static void dbengine2_statistics_charts(void) {
         static RRDDIM *rd_uncompressed = NULL;
         static RRDDIM *rd_mmap_failed = NULL;
         static RRDDIM *rd_unavailable = NULL;
-        static RRDDIM *rd_already_loaded = NULL;
-        static RRDDIM *rd_preloaded = NULL;
         static RRDDIM *rd_unroutable = NULL;
         static RRDDIM *rd_not_found = NULL;
         static RRDDIM *rd_invalid_extent = NULL;
+        static RRDDIM *rd_extent_merged = NULL;
 
         if (unlikely(!st_query_pages_from_disk)) {
             st_query_pages_from_disk = rrdset_create_localhost(
@@ -2002,12 +1994,11 @@ static void dbengine2_statistics_charts(void) {
             rd_invalid = rrddim_add(st_query_pages_from_disk, "fail invalid page", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_uncompressed = rrddim_add(st_query_pages_from_disk, "ok uncompressed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_mmap_failed = rrddim_add(st_query_pages_from_disk, "fail cant mmap", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_already_loaded = rrddim_add(st_query_pages_from_disk, "ok but preloaded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_unavailable = rrddim_add(st_query_pages_from_disk, "fail unavailable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_preloaded = rrddim_add(st_query_pages_from_disk, "ok preloaded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_unroutable = rrddim_add(st_query_pages_from_disk, "fail unroutable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_not_found = rrddim_add(st_query_pages_from_disk, "fail uuid not found", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_invalid_extent = rrddim_add(st_query_pages_from_disk, "fail invalid extent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_extent_merged = rrddim_add(st_query_pages_from_disk, "extent merged", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
         }
         priority++;
 
@@ -2015,12 +2006,11 @@ static void dbengine2_statistics_charts(void) {
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_invalid, (collected_number)cache_efficiency_stats.pages_load_fail_invalid_page_in_extent);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_uncompressed, (collected_number)cache_efficiency_stats.pages_load_ok_uncompressed);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_mmap_failed, (collected_number)cache_efficiency_stats.pages_load_fail_cant_mmap_extent);
-        rrddim_set_by_pointer(st_query_pages_from_disk, rd_already_loaded, (collected_number)cache_efficiency_stats.pages_load_ok_loaded_but_cache_hit_before_allocation);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_unavailable, (collected_number)cache_efficiency_stats.pages_load_fail_datafile_not_available);
-        rrddim_set_by_pointer(st_query_pages_from_disk, rd_preloaded, (collected_number)cache_efficiency_stats.pages_load_ok_preloaded);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_unroutable, (collected_number)cache_efficiency_stats.pages_load_fail_unroutable);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_not_found, (collected_number)cache_efficiency_stats.pages_load_fail_uuid_not_found);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_invalid_extent, (collected_number)cache_efficiency_stats.pages_load_fail_invalid_extent);
+        rrddim_set_by_pointer(st_query_pages_from_disk, rd_extent_merged, (collected_number)cache_efficiency_stats.pages_load_extent_merged);
 
         rrdset_done(st_query_pages_from_disk);
     }