Optimizations part 3 (#15293)

* use madvise to speed up indexing

* collect all rrddim members into a collector structure

* use tier 0 virtual point for storing last stored value

* reorganize key fields in rrddim

* remove fgets from pluginsd and replace it with read()

* properly uncork the web server sockets

* Revert "reorganize key fields in rrddim"

This reverts commit 2d45fa3959087e05462d387ff115a260f3a04b60.

* Revert "use tier 0 virtual point for storing last stored value"

This reverts commit a576cdd377ad4778a3b8608cabbb7ea7bb19a3a8.

* fix cork names

* fix compilation warnings

1 files changed, 12 insertions, 7 deletions

diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c
index 234505cb19..9832b5784c 100644
--- a/database/engine/journalfile.c
+++ b/database/engine/journalfile.c
@@ -216,15 +216,20 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin
             madvise_dontfork(journalfile->mmap.data, journalfile->mmap.size);
             madvise_dontdump(journalfile->mmap.data, journalfile->mmap.size);
 
-            // let the kernel know that we don't want read-ahead on this file
-            madvise_random(journalfile->mmap.data, journalfile->mmap.size);
-
-//            madvise_willneed(journalfile->mmap.data, journalfile->v2.size_of_directory);
-//            madvise_dontneed(journalfile->mmap.data, journalfile->mmap.size);
-
             spinlock_lock(&journalfile->v2.spinlock);
             journalfile->v2.flags |= JOURNALFILE_FLAG_IS_AVAILABLE | JOURNALFILE_FLAG_IS_MOUNTED;
+            JOURNALFILE_FLAGS flags = journalfile->v2.flags;
             spinlock_unlock(&journalfile->v2.spinlock);
+
+            if(flags & JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION) {
+                // we need the entire metrics directory into memory to process it
+                madvise_willneed(journalfile->mmap.data, journalfile->v2.size_of_directory);
+            }
+            else {
+                // let the kernel know that we don't want read-ahead on this file
+                madvise_random(journalfile->mmap.data, journalfile->mmap.size);
+                // madvise_dontneed(journalfile->mmap.data, journalfile->mmap.size);
+            }
         }
     }
 
@@ -413,7 +418,7 @@ void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd,
     struct journal_v2_header *j2_header = journalfile->mmap.data;
     journalfile->v2.first_time_s = (time_t)(j2_header->start_time_ut / USEC_PER_SEC);
     journalfile->v2.last_time_s = (time_t)(j2_header->end_time_ut / USEC_PER_SEC);
-    // journalfile->v2.size_of_directory = j2_header->metric_offset + j2_header->metric_count * sizeof(struct journal_metric_list);
+    journalfile->v2.size_of_directory = j2_header->metric_offset + j2_header->metric_count * sizeof(struct journal_metric_list);
 
     journalfile_v2_mounted_data_unmount(journalfile, true, true);