DBENGINE v2 - improvements 2 (#14257)

* allow extents to be merged for as long as possible

* do not block the event loop while recalculating retention due to datafile rotation

* buffers are incrementally cleaned up, every second, by just 1 entry

* fix order of commands

* remove newline

* measure cancelled extent read requests

* count all cancelled extent requests

* do not double count failed pages

* fixed cancelled name

* Fix error and warnings when compiling with --disable-dbengine

* when the timeframe is outside retention and whole query should fail

* do not mark as failed pages that have been loaded but have been skipped

* added chart to show cache memory calculation variables

* LONG_MAX for 32-bit compatibility

* fix cache size calculation on 32-bit

* fix cache size calculation on 32-bit - use unsinged long long

* fix compilation warnings on 32-bits

* fix another compilation warning on 32-bits

* fix compilation warnings on older 32-bit compilers

* fix compilation warnings on older 32-bit compilers - more of them

* disable ML threads joining

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 66 insertions, 3 deletions

diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 40e8795b82..b519e70511 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -1023,6 +1023,15 @@ struct dbengine2_cache_pointers {
     RRDDIM *rd_pgc_memory_evicting;
     RRDDIM *rd_pgc_memory_flushing;
 
+    RRDSET *st_pgc_tm;
+    RRDDIM *rd_pgc_tm_current;
+    RRDDIM *rd_pgc_tm_wanted;
+    RRDDIM *rd_pgc_tm_hot_max;
+    RRDDIM *rd_pgc_tm_dirty_max;
+    RRDDIM *rd_pgc_tm_hot;
+    RRDDIM *rd_pgc_tm_dirty;
+    RRDDIM *rd_pgc_tm_referenced;
+
     RRDSET *st_pgc_pages;
     RRDDIM *rd_pgc_pages_clean;
     RRDDIM *rd_pgc_pages_hot;
@@ -1168,7 +1177,6 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
     }
 
     {
-
         if (unlikely(!ptrs->st_pgc_memory)) {
             BUFFER *id = buffer_create(100);
             buffer_sprintf(id, "dbengine_%s_cache_memory", name);
@@ -1223,6 +1231,56 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p
     }
 
     {
+        if (unlikely(!ptrs->st_pgc_tm)) {
+            BUFFER *id = buffer_create(100);
+            buffer_sprintf(id, "dbengine_%s_target_memory", name);
+
+            BUFFER *family = buffer_create(100);
+            buffer_sprintf(family, "dbengine %s cache", name);
+
+            BUFFER *title = buffer_create(100);
+            buffer_sprintf(title, "Netdata %s Target Cache Memory", name);
+
+            ptrs->st_pgc_tm = rrdset_create_localhost(
+                    "netdata",
+                    buffer_tostring(id),
+                    NULL,
+                    buffer_tostring(family),
+                    NULL,
+                    buffer_tostring(title),
+                    "bytes",
+                    "netdata",
+                    "stats",
+                    priority,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_LINE);
+
+            ptrs->rd_pgc_tm_current    = rrddim_add(ptrs->st_pgc_tm, "current",    NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_wanted     = rrddim_add(ptrs->st_pgc_tm, "wanted",     NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_referenced = rrddim_add(ptrs->st_pgc_tm, "referenced", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_hot_max    = rrddim_add(ptrs->st_pgc_tm, "hot max",    NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_dirty_max  = rrddim_add(ptrs->st_pgc_tm, "dirty max",  NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_hot        = rrddim_add(ptrs->st_pgc_tm, "hot",        NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            ptrs->rd_pgc_tm_dirty      = rrddim_add(ptrs->st_pgc_tm, "dirty",      NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+            buffer_free(id);
+            buffer_free(family);
+            buffer_free(title);
+            priority++;
+        }
+
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_current, (collected_number)pgc_stats->current_cache_size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_wanted, (collected_number)pgc_stats->wanted_cache_size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_referenced, (collected_number)pgc_stats->referenced_size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_hot_max, (collected_number)pgc_stats->queues.hot.max_size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_dirty_max, (collected_number)pgc_stats->queues.dirty.max_size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_hot, (collected_number)pgc_stats->queues.hot.size);
+        rrddim_set_by_pointer(ptrs->st_pgc_tm, ptrs->rd_pgc_tm_dirty, (collected_number)pgc_stats->queues.dirty.size);
+
+        rrdset_done(ptrs->st_pgc_tm);
+    }
+
+    {
         if (unlikely(!ptrs->st_pgc_pages)) {
             BUFFER *id = buffer_create(100);
             buffer_sprintf(id, "dbengine_%s_cache_pages", name);
@@ -1972,6 +2030,7 @@ static void dbengine2_statistics_charts(void) {
         static RRDDIM *rd_unavailable = NULL;
         static RRDDIM *rd_unroutable = NULL;
         static RRDDIM *rd_not_found = NULL;
+        static RRDDIM *rd_cancelled = NULL;
         static RRDDIM *rd_invalid_extent = NULL;
         static RRDDIM *rd_extent_merged = NULL;
 
@@ -1996,9 +2055,10 @@ static void dbengine2_statistics_charts(void) {
             rd_mmap_failed = rrddim_add(st_query_pages_from_disk, "fail cant mmap", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_unavailable = rrddim_add(st_query_pages_from_disk, "fail unavailable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_unroutable = rrddim_add(st_query_pages_from_disk, "fail unroutable", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_not_found = rrddim_add(st_query_pages_from_disk, "fail uuid not found", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_not_found = rrddim_add(st_query_pages_from_disk, "fail not found", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_invalid_extent = rrddim_add(st_query_pages_from_disk, "fail invalid extent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
             rd_extent_merged = rrddim_add(st_query_pages_from_disk, "extent merged", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_cancelled = rrddim_add(st_query_pages_from_disk, "cancelled", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
         }
         priority++;
 
@@ -2008,7 +2068,8 @@ static void dbengine2_statistics_charts(void) {
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_mmap_failed, (collected_number)cache_efficiency_stats.pages_load_fail_cant_mmap_extent);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_unavailable, (collected_number)cache_efficiency_stats.pages_load_fail_datafile_not_available);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_unroutable, (collected_number)cache_efficiency_stats.pages_load_fail_unroutable);
-        rrddim_set_by_pointer(st_query_pages_from_disk, rd_not_found, (collected_number)cache_efficiency_stats.pages_load_fail_uuid_not_found);
+        rrddim_set_by_pointer(st_query_pages_from_disk, rd_not_found, (collected_number)cache_efficiency_stats.pages_load_fail_not_found);
+        rrddim_set_by_pointer(st_query_pages_from_disk, rd_cancelled, (collected_number)cache_efficiency_stats.pages_load_fail_cancelled);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_invalid_extent, (collected_number)cache_efficiency_stats.pages_load_fail_invalid_extent);
         rrddim_set_by_pointer(st_query_pages_from_disk, rd_extent_merged, (collected_number)cache_efficiency_stats.pages_load_extent_merged);
 
@@ -3778,10 +3839,12 @@ void *global_statistics_main(void *ptr)
         worker_is_busy(WORKER_JOB_REGISTRY);
         registry_statistics();
 
+#ifdef ENABLE_DBENGINE
         if(dbengine_enabled) {
             worker_is_busy(WORKER_JOB_DBENGINE);
             dbengine2_statistics_charts();
         }
+#endif
 
         worker_is_busy(WORKER_JOB_HEARTBEAT);
         update_heartbeat_charts();