DBENGINE v2 - improvements part 3 (#14269)

* reduce journal v2 shared memory using madvise() - not integrated yet

* working attempt to minimize dbengine shared memory

* never call willneed - let the kernel decide which parts of each file are really needed

* journal files get MADV_RANDOM

* dont call MADV_DONTNEED too frequently

* madvise() is always called with the journal unlocked but referenced

* call madvise() even less frequently

* added chart for monitoring database events

* turn batch mode on under critical conditions

* max size to evict is 1/4 of the max

* fix max size to evict calculation

* use dbengine_page/extent_alloc/free to pages and extents allocations, tracking also the size of these allocations at free time

* fix calculation for batch evictions

* allow main and open cache to have as many evictors as needed

* control inline evictors for each cache; report different levels of cache pressure on every cache evaluation

* more inline evictors for extent cache

* bypass max inline evictors above critical level

* current cache usage has to be taken

* re-arrange items in journafile

* updated docs - work in progress

* more docs work

* more docs work

* Map / unmap journal file

* draw.io diagram for dbengine operations

* updated dbengine diagram

* updated docs

* journal files v2 now get mapped and unmapped as needed

* unmap journal v2 immediately when getting retention

* mmap and munmap do not block queries evaluating journal files v2

* have only one unmap function

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

3 files changed, 29 insertions, 6 deletions

diff --git a/libnetdata/arrayalloc/arrayalloc.c b/libnetdata/arrayalloc/arrayalloc.c
index 57439e34d9..090e72f363 100644
--- a/libnetdata/arrayalloc/arrayalloc.c
+++ b/libnetdata/arrayalloc/arrayalloc.c
@@ -209,7 +209,7 @@ static void arrayalloc_add_page(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_P
         char filename[FILENAME_MAX + 1];
         snprintfz(filename, FILENAME_MAX, "%s/array_alloc.mmap/%s.%zu", *ar->cache_dir, ar->filename, ar->internal.file_number);
         page->filename = strdupz(filename);
-        page->data = netdata_mmap(page->filename, page->size, MAP_SHARED, 0, false);
+        page->data = netdata_mmap(page->filename, page->size, MAP_SHARED, 0, false, NULL);
         if (unlikely(!page->data))
             fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename);
     }
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index 210d742ffc..aa7a6d7364 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -1184,6 +1184,14 @@ inline int madvise_sequential(void *mem, size_t len) {
     return ret;
 }
 
+inline int madvise_random(void *mem, size_t len) {
+    static int logger = 1;
+    int ret = madvise(mem, len, MADV_RANDOM);
+
+    if (ret != 0 && logger-- > 0) error("madvise(MADV_RANDOM) failed.");
+    return ret;
+}
+
 inline int madvise_dontfork(void *mem, size_t len) {
     static int logger = 1;
     int ret = madvise(mem, len, MADV_DONTFORK);
@@ -1200,6 +1208,14 @@ inline int madvise_willneed(void *mem, size_t len) {
     return ret;
 }
 
+inline int madvise_dontneed(void *mem, size_t len) {
+    static int logger = 1;
+    int ret = madvise(mem, len, MADV_DONTNEED);
+
+    if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTNEED) failed.");
+    return ret;
+}
+
 inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) {
 #if __linux__
     static int logger = 1;
@@ -1224,7 +1240,7 @@ inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused
 #endif
 }
 
-void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only)
+void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, int *open_fd)
 {
     // info("netdata_mmap('%s', %zu", filename, size);
 
@@ -1281,15 +1297,20 @@ void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool r
             else info("Cannot seek to beginning of file '%s'.", filename);
         }
 
-        madvise_sequential(mem, size);
+        // madvise_sequential(mem, size);
         madvise_dontfork(mem, size);
         madvise_dontdump(mem, size);
-        if(flags & MAP_SHARED) madvise_willneed(mem, size);
+        // if(flags & MAP_SHARED) madvise_willneed(mem, size);
         if(ksm) madvise_mergeable(mem, size);
     }
 
 cleanup:
-    if(fd != -1) close(fd);
+    if(fd != -1) {
+        if (open_fd)
+            *open_fd = fd;
+        else
+            close(fd);
+    }
     if(mem == MAP_FAILED) return NULL;
     errno = 0;
     return mem;
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index 5abe6eb854..7a7fe032ff 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -308,8 +308,10 @@ char *trim(char *s); // remove leading and trailing spaces; may return NULL
 char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL
 
 int madvise_sequential(void *mem, size_t len);
+int madvise_random(void *mem, size_t len);
 int madvise_dontfork(void *mem, size_t len);
 int madvise_willneed(void *mem, size_t len);
+int madvise_dontneed(void *mem, size_t len);
 int madvise_dontdump(void *mem, size_t len);
 int madvise_mergeable(void *mem, size_t len);
 
@@ -347,7 +349,7 @@ void posix_memfree(void *ptr);
 void json_escape_string(char *dst, const char *src, size_t size);
 void json_fix_string(char *s);
 
-void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only);
+void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, int *open_fd);
 int netdata_munmap(void *ptr, size_t size);
 int memory_file_save(const char *filename, void *mem, size_t size);