Make sure we open as many files as possible in the recursion

Signed-off-by: Matthias Beyer <matthias.beyer@atos.net>

2 files changed, 14 insertions, 1 deletions

diff --git a/Cargo.toml b/Cargo.toml
index 1222869..c8c186b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,6 +56,7 @@ regex          = "1"
 reqwest        = { version = "0.11", features = [ "stream" ] }
 resiter        = "0.4"
 result-inspect = "0.2"
+rlimit         = "0.6"
 semver		   = { version = "1.0", features = [ "serde" ] }
 serde          = "1"
 serde_json     = "1"
diff --git a/src/repository/fs.rs b/src/repository/fs.rs
index 32b7e6e..ce5531c 100644
--- a/src/repository/fs.rs
+++ b/src/repository/fs.rs
@@ -5,6 +5,7 @@ use std::path::PathBuf;
 use std::collections::HashMap;
 use std::path::Component;
 use std::convert::TryFrom;
+use std::convert::TryInto;
 
 use walkdir::DirEntry;
 use walkdir::WalkDir;
@@ -91,10 +92,21 @@ impl FileSystemRepresentation {
             files: vec![],
         };
 
+        let max_files_open = {
+            let (soft, _hard) = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
+
+            // use less than the soft limit if the soft limit is above 15
+            soft.checked_sub(16)
+                .unwrap_or(soft)
+                .try_into() // we need to have a usize
+                .unwrap_or(usize::MAX) // if usize is smaller than u64, usize::MAX will do
+        };
+
         log::trace!("Loading files from filesystem starting at: {}", root.display());
+        log::trace!("Loading with a maximum of {} files open", max_files_open);
         WalkDir::new(root)
             .follow_links(false)
-            .max_open(100)
+            .max_open(max_files_open)
             .same_file_system(true)
             .into_iter()
             .filter_entry(|e| !is_hidden(e) && (is_pkgtoml(e) || is_dir(e)))