better daemon errors about files; (#4342)

* better daemon errors about files; fixes #4341

* allow max depth 10 while reading dirs; fixes #4312

* exclude . and .. even if they are not reported as directories; #4312

* lower recursive depth to 3

* strict control on directory recursion

8 files changed, 101 insertions, 65 deletions

diff --git a/src/appconfig.c b/src/appconfig.c
index 5c7193f651..23f73fa698 100644
--- a/src/appconfig.c
+++ b/src/appconfig.c
@@ -447,7 +447,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
 
     FILE *fp = fopen(filename, "r");
     if(!fp) {
-        error("CONFIG: cannot open file '%s'", filename);
+        // info("CONFIG: cannot open file '%s'. Using internal defaults.", filename);
         return 0;
     }
 
diff --git a/src/apps_plugin.c b/src/apps_plugin.c
index 1dc3b711ec..dc08ada5de 100644
--- a/src/apps_plugin.c
+++ b/src/apps_plugin.c
@@ -3571,7 +3571,7 @@ static void parse_args(int argc, char **argv)
     if(freq > 0) update_every = freq;
 
     if(read_apps_groups_conf(user_config_dir, "groups")) {
-        error("Cannot read process groups configuration file '%s/apps_groups.conf'. Will try '%s/apps_groups.conf'", user_config_dir, stock_config_dir);
+        info("Cannot read process groups configuration file '%s/apps_groups.conf'. Will try '%s/apps_groups.conf'", user_config_dir, stock_config_dir);
 
         if(read_apps_groups_conf(stock_config_dir, "groups")) {
             error("Cannot read process groups '%s/apps_groups.conf'. There are no internal defaults. Failing.", stock_config_dir);
diff --git a/src/common.c b/src/common.c
index 363ae2455a..80e570ffdf 100644
--- a/src/common.c
+++ b/src/common.c
@@ -1534,84 +1534,95 @@ int path_is_file(const char *path, const char *subpath) {
     return is_file;
 }
 
-void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data) {
+void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) {
+    if(depth > 3) {
+        error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath);
+        return;
+    }
+
     char *udir = strdupz_path_subpath(user_path, subpath);
     char *sdir = strdupz_path_subpath(stock_path, subpath);
 
-    debug(D_HEALTH, "Configuration traversing user-config directory '%s', stock config directory '%s'", udir, sdir);
+    debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir);
 
     DIR *dir = opendir(udir);
     if (!dir) {
-        error("Configuration cannot open user-config directory '%s'.", udir);
+        error("CONFIG cannot open user-config directory '%s'.", udir);
     }
     else {
         struct dirent *de = NULL;
         while((de = readdir(dir))) {
-            if(de->d_type == DT_DIR
-               && (   (de->d_name[0] == '.' && de->d_name[1] == '\0')
-                      || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
-               )) {
-                debug(D_HEALTH, "Configuration ignoring user-config directory '%s/%s'", udir, de->d_name);
-                continue;
+            if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+                if( !de->d_name[0] ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+                        ) {
+                    debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name);
+                    continue;
+                }
+
+                if(path_is_dir(udir, de->d_name)) {
+                    recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+                    continue;
+                }
             }
 
-            if(path_is_dir(udir, de->d_name)) {
-                recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data);
-                continue;
+            if(de->d_type == DT_REG || de->d_type == DT_LNK) {
+                size_t len = strlen(de->d_name);
+                if(path_is_file(udir, de->d_name) &&
+                   len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+                    char *filename = strdupz_path_subpath(udir, de->d_name);
+                    callback(filename, data);
+                    freez(filename);
+                }
+                else
+                    debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s'", udir, de->d_name);
             }
-
-            size_t len = strlen(de->d_name);
-            if(path_is_file(udir, de->d_name) &&
-               len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
-                char *filename = strdupz_path_subpath(udir, de->d_name);
-                callback(filename, data);
-                freez(filename);
-            }
-
-            else
-                debug(D_HEALTH, "Health ignoring user config file '%s/%s'", udir, de->d_name);
         }
 
         closedir(dir);
     }
 
-    debug(D_HEALTH, "Health configuration traversing stock config directory '%s', user config directory '%s'", sdir, udir);
+    debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir);
 
     dir = opendir(sdir);
     if (!dir) {
-        error("Health configuration cannot open stock config directory '%s'.", sdir);
+        error("CONFIG cannot open stock config directory '%s'.", sdir);
     }
     else {
         struct dirent *de = NULL;
         while((de = readdir(dir))) {
-            if(de->d_type == DT_DIR
-               && (      (de->d_name[0] == '.' && de->d_name[1] == '\0')
-                         || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
-               )) {
-                debug(D_HEALTH, "Health ignoring stock config directory '%s/%s'", sdir, de->d_name);
-                continue;
+            if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+                if( !de->d_name[0] ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+                        ) {
+                    debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name);
+                    continue;
+                }
+
+                if(path_is_dir(sdir, de->d_name)) {
+                    // we recurse in stock subdirectory, only when there is no corresponding
+                    // user subdirectory - to avoid reading the files twice
+
+                    if(!path_is_dir(udir, de->d_name))
+                        recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+
+                    continue;
+                }
             }
 
-            if(path_is_dir(sdir, de->d_name)) {
-                // we recurse in stock subdirectory, only when there is no corresponding
-                // user subdirectory - to avoid reading the files twice
-
-                if(!path_is_dir(udir, de->d_name))
-                    recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data);
-
-                continue;
+            if(de->d_type == DT_REG || de->d_type == DT_LNK) {
+                size_t len = strlen(de->d_name);
+                if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) &&
+                   len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+                    char *filename = strdupz_path_subpath(sdir, de->d_name);
+                    callback(filename, data);
+                    freez(filename);
+                }
+                else
+                    debug(D_HEALTH, "CONFIG ignoring stock config file '%s/%s'", sdir, de->d_name);
             }
-
-            size_t len = strlen(de->d_name);
-            if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) &&
-               len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
-                char *filename = strdupz_path_subpath(sdir, de->d_name);
-                callback(filename, data);
-                freez(filename);
-            }
-
-            else
-                debug(D_HEALTH, "Health ignoring stock config file '%s/%s'", sdir, de->d_name);
         }
 
         closedir(dir);
diff --git a/src/common.h b/src/common.h
index 0c1191febb..e231d3ee96 100644
--- a/src/common.h
+++ b/src/common.h
@@ -431,8 +431,14 @@ extern const char *program_version;
 extern char *strdupz_path_subpath(const char *path, const char *subpath);
 extern int path_is_dir(const char *path, const char *subpath);
 extern int path_is_file(const char *path, const char *subpath);
-extern void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath
-                                             , int (*callback)(const char *filename, void *data), void *data);
+extern void recursive_config_double_dir_load(
+          const char *user_path
+        , const char *stock_path
+        , const char *subpath
+        , int (*callback)(const char *filename, void *data)
+        , void *data
+        , size_t depth
+        );
 
 /* fix for alpine linux */
 #ifndef RUSAGE_THREAD
diff --git a/src/health_config.c b/src/health_config.c
index 618f071711..6aaf1af18c 100644
--- a/src/health_config.c
+++ b/src/health_config.c
@@ -855,5 +855,5 @@ static int health_readfile(const char *filename, void *data) {
 
 void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) {
     if(unlikely(!host->health_enabled)) return;
-    recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host);
+    recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host, 0);
 }
diff --git a/src/log.h b/src/log.h
index f1b7177d05..d5faac6e90 100644
--- a/src/log.h
+++ b/src/log.h
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
+#include "common.h"
+
 #ifndef NETDATA_LOG_H
 #define NETDATA_LOG_H 1
 
diff --git a/src/main.c b/src/main.c
index 367e402491..b80ecccf75 100644
--- a/src/main.c
+++ b/src/main.c
@@ -630,28 +630,45 @@ void set_global_environment() {
 }
 
 static int load_netdata_conf(char *filename, char overwrite_used) {
-    if(filename)
-        return config_load(filename, overwrite_used);
+    errno = 0;
 
-    filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf");
+    int ret = 0;
 
-    int ret = config_load(filename, overwrite_used);
-    if(!ret) {
-        freez(filename);
-        filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf");
+    if(filename && *filename) {
         ret = config_load(filename, overwrite_used);
+        if(!ret)
+            error("CONFIG: cannot load config file '%s'.", filename);
+    }
+    else {
+        filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf");
+
+        ret = config_load(filename, overwrite_used);
+        if(!ret) {
+            info("CONFIG: cannot load user config '%s'. Will try the stock version.", filename);
+            freez(filename);
+
+            filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf");
+            ret = config_load(filename, overwrite_used);
+            if(!ret)
+                info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
+        }
+
+        freez(filename);
     }
-    freez(filename);
 
     return ret;
 }
 
 static void load_stream_conf() {
+    errno = 0;
     char *filename = strdupz_path_subpath(netdata_configured_user_config_dir, "stream.conf");
     if(!appconfig_load(&stream_config, filename, 0)) {
+        info("CONFIG: cannot load user config '%s'. Will try stock config.", filename);
         freez(filename);
+
         filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "stream.conf");
-        appconfig_load(&stream_config, filename, 0);
+        if(!appconfig_load(&stream_config, filename, 0))
+            info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
     }
     freez(filename);
 }
diff --git a/src/statsd.c b/src/statsd.c
index 9237ba8f71..4e2f487cf1 100644
--- a/src/statsd.c
+++ b/src/statsd.c
@@ -1392,7 +1392,7 @@ static int statsd_file_callback(const char *filename, void *data) {
 }
 
 static inline void statsd_readdir(const char *user_path, const char *stock_path, const char *subpath) {
-    recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL);
+    recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL, 0);
 }
 
 // --------------------------------------------------------------------------------------------------------------------