DBENGINE v2 - improvements part 3 (#14269)

* reduce journal v2 shared memory using madvise() - not integrated yet

* working attempt to minimize dbengine shared memory

* never call willneed - let the kernel decide which parts of each file are really needed

* journal files get MADV_RANDOM

* dont call MADV_DONTNEED too frequently

* madvise() is always called with the journal unlocked but referenced

* call madvise() even less frequently

* added chart for monitoring database events

* turn batch mode on under critical conditions

* max size to evict is 1/4 of the max

* fix max size to evict calculation

* use dbengine_page/extent_alloc/free to pages and extents allocations, tracking also the size of these allocations at free time

* fix calculation for batch evictions

* allow main and open cache to have as many evictors as needed

* control inline evictors for each cache; report different levels of cache pressure on every cache evaluation

* more inline evictors for extent cache

* bypass max inline evictors above critical level

* current cache usage has to be taken

* re-arrange items in journafile

* updated docs - work in progress

* more docs work

* more docs work

* Map / unmap journal file

* draw.io diagram for dbengine operations

* updated dbengine diagram

* updated docs

* journal files v2 now get mapped and unmapped as needed

* unmap journal v2 immediately when getting retention

* mmap and munmap do not block queries evaluating journal files v2

* have only one unmap function

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

1 files changed, 46 insertions, 0 deletions

diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index b519e70511..3af4368c48 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -2077,6 +2077,52 @@ static void dbengine2_statistics_charts(void) {
     }
 
     {
+        static RRDSET *st_events = NULL;
+        static RRDDIM *rd_journal_v2_mapped = NULL;
+        static RRDDIM *rd_journal_v2_unmapped = NULL;
+        static RRDDIM *rd_datafile_creation = NULL;
+        static RRDDIM *rd_datafile_deletion = NULL;
+        static RRDDIM *rd_datafile_deletion_spin = NULL;
+        static RRDDIM *rd_jv2_indexing = NULL;
+        static RRDDIM *rd_retention = NULL;
+
+        if (unlikely(!st_events)) {
+            st_events = rrdset_create_localhost(
+                    "netdata",
+                    "dbengine_events",
+                    NULL,
+                    "dbengine query router",
+                    NULL,
+                    "Netdata Database Events",
+                    "events/s",
+                    "netdata",
+                    "stats",
+                    priority,
+                    localhost->rrd_update_every,
+                    RRDSET_TYPE_LINE);
+
+            rd_journal_v2_mapped = rrddim_add(st_events, "journal v2 mapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_journal_v2_unmapped = rrddim_add(st_events, "journal v2 unmapped", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_datafile_creation = rrddim_add(st_events, "datafile creation", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_datafile_deletion = rrddim_add(st_events, "datafile deletion", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_datafile_deletion_spin = rrddim_add(st_events, "datafile deletion spin", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_jv2_indexing = rrddim_add(st_events, "journal v2 indexing", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            rd_retention = rrddim_add(st_events, "retention", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+        }
+        priority++;
+
+        rrddim_set_by_pointer(st_events, rd_journal_v2_mapped, (collected_number)cache_efficiency_stats.journal_v2_mapped);
+        rrddim_set_by_pointer(st_events, rd_journal_v2_unmapped, (collected_number)cache_efficiency_stats.journal_v2_unmapped);
+        rrddim_set_by_pointer(st_events, rd_datafile_creation, (collected_number)cache_efficiency_stats.datafile_creation_started);
+        rrddim_set_by_pointer(st_events, rd_datafile_deletion, (collected_number)cache_efficiency_stats.datafile_deletion_started);
+        rrddim_set_by_pointer(st_events, rd_datafile_deletion_spin, (collected_number)cache_efficiency_stats.datafile_deletion_spin);
+        rrddim_set_by_pointer(st_events, rd_jv2_indexing, (collected_number)cache_efficiency_stats.journal_v2_indexing_started);
+        rrddim_set_by_pointer(st_events, rd_retention, (collected_number)cache_efficiency_stats.metrics_retention_started);
+
+        rrdset_done(st_events);
+    }
+
+    {
         static RRDSET *st_prep_timings = NULL;
         static RRDDIM *rd_routing = NULL;
         static RRDDIM *rd_main_cache = NULL;