Improve the impact of health code on netdata scalability (#8407)

* Add support for spawning processes without pipes.

* Port health_alarm_execute() from mypopen() to netdata_spawn()

* Make alarm notifications asynchronous within a single health thread iteration

* Initial version of spawn server.

* preliminary integration of spawn client with health

2 files changed, 86 insertions, 34 deletions

diff --git a/libnetdata/popen/popen.c b/libnetdata/popen/popen.c
index 1c4ae64d6c..c0135cf406 100644
--- a/libnetdata/popen/popen.c
+++ b/libnetdata/popen/popen.c
@@ -78,8 +78,16 @@ static void myp_del(pid_t pid) {
 #define PIPE_READ 0
 #define PIPE_WRITE 1
 
-static inline FILE *custom_popene(const char *command, volatile pid_t *pidptr, char **env) {
-    FILE *fp;
+/* custom_popene flag definitions */
+#define FLAG_CREATE_PIPE    1 // Create a pipe like popen() when set, otherwise set stdout to /dev/null
+#define FLAG_CLOSE_FD       2 // Close all file descriptors other than STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO
+
+/*
+ * Returns -1 on failure, 0 on success. When FLAG_CREATE_PIPE is set, on success set the FILE *fp pointer.
+ */
+static inline int custom_popene(const char *command, volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp) {
+    FILE *fp = NULL;
+    int ret = 0; // success by default
     int pipefd[2], error;
     pid_t pid;
     char *const spawn_argv[] = {
@@ -91,23 +99,36 @@ static inline FILE *custom_popene(const char *command, volatile pid_t *pidptr, c
     posix_spawnattr_t attr;
     posix_spawn_file_actions_t fa;
 
-    if (pipe(pipefd) == -1)
-        return NULL;
-    if ((fp = fdopen(pipefd[PIPE_READ], "r")) == NULL) {
-        goto error_after_pipe;
+    if (flags & FLAG_CREATE_PIPE) {
+        if (pipe(pipefd) == -1)
+            return -1;
+        if ((fp = fdopen(pipefd[PIPE_READ], "r")) == NULL) {
+            goto error_after_pipe;
+        }
     }
 
-    // Mark all files to be closed by the exec() stage of posix_spawn()
-    int i;
-    for (i = (int) (sysconf(_SC_OPEN_MAX) - 1); i >= 0; i--)
-        if(i != STDIN_FILENO && i != STDERR_FILENO)
-            (void)fcntl(i, F_SETFD, FD_CLOEXEC);
+    if (flags & FLAG_CLOSE_FD) {
+        // Mark all files to be closed by the exec() stage of posix_spawn()
+        int i;
+        for (i = (int) (sysconf(_SC_OPEN_MAX) - 1); i >= 0; i--) {
+            if (i != STDIN_FILENO && i != STDERR_FILENO)
+                (void) fcntl(i, F_SETFD, FD_CLOEXEC);
+        }
+    }
 
     if (!posix_spawn_file_actions_init(&fa)) {
-        // move the pipe to stdout in the child
-        if (posix_spawn_file_actions_adddup2(&fa, pipefd[PIPE_WRITE], STDOUT_FILENO)) {
-            error("posix_spawn_file_actions_adddup2() failed");
-            goto error_after_posix_spawn_file_actions_init;
+        if (flags & FLAG_CREATE_PIPE) {
+            // move the pipe to stdout in the child
+            if (posix_spawn_file_actions_adddup2(&fa, pipefd[PIPE_WRITE], STDOUT_FILENO)) {
+                error("posix_spawn_file_actions_adddup2() failed");
+                goto error_after_posix_spawn_file_actions_init;
+            }
+        } else {
+            // set stdout to /dev/null
+            if (posix_spawn_file_actions_addopen(&fa, STDOUT_FILENO, "/dev/null", O_WRONLY, 0)) {
+                error("posix_spawn_file_actions_addopen() failed");
+                // this is not a fatal error
+            }
         }
     } else {
         error("posix_spawn_file_actions_init() failed.");
@@ -136,10 +157,16 @@ static inline FILE *custom_popene(const char *command, volatile pid_t *pidptr, c
     } else {
         myp_add_unlock();
         error("Failed to spawn command: '%s' from parent pid %d.", command, getpid());
-        fclose(fp);
-        fp = NULL;
+        if (flags & FLAG_CREATE_PIPE) {
+            fclose(fp);
+        }
+        ret = -1;
+    }
+    if (flags & FLAG_CREATE_PIPE) {
+        close(pipefd[PIPE_WRITE]);
+        if (0 == ret) // on success set FILE * pointer
+            *fpp = fp;
     }
-    close(pipefd[PIPE_WRITE]);
 
     if (!error) {
         // posix_spawnattr_init() succeeded
@@ -149,19 +176,21 @@ static inline FILE *custom_popene(const char *command, volatile pid_t *pidptr, c
     if (posix_spawn_file_actions_destroy(&fa))
         error("posix_spawn_file_actions_destroy");
 
-    return fp;
+    return ret;
 
 error_after_posix_spawn_file_actions_init:
     if (posix_spawn_file_actions_destroy(&fa))
         error("posix_spawn_file_actions_destroy");
 error_after_pipe:
-    if (fp)
-        fclose(fp);
-    else
-        close(pipefd[PIPE_READ]);
+    if (flags & FLAG_CREATE_PIPE) {
+        if (fp)
+            fclose(fp);
+        else
+            close(pipefd[PIPE_READ]);
 
-    close(pipefd[PIPE_WRITE]);
-    return NULL;
+        close(pipefd[PIPE_WRITE]);
+    }
+    return -1;
 }
 
 // See man environ
@@ -222,26 +251,37 @@ int myp_reap(pid_t pid) {
 }
 
 FILE *mypopen(const char *command, volatile pid_t *pidptr) {
-    return custom_popene(command, pidptr, environ);
+    FILE *fp = NULL;
+    (void)custom_popene(command, pidptr, environ, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp);
+    return fp;
 }
 
 FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env) {
-    return custom_popene(command, pidptr, env);
+    FILE *fp = NULL;
+    (void)custom_popene(command, pidptr, env, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp);
+    return fp;
+}
+
+// returns 0 on success, -1 on failure
+int netdata_spawn(const char *command, volatile pid_t *pidptr) {
+    return custom_popene(command, pidptr, environ, 0, NULL);
 }
 
-int mypclose(FILE *fp, pid_t pid) {
+int custom_pclose(FILE *fp, pid_t pid) {
     int ret;
     siginfo_t info;
 
     debug(D_EXIT, "Request to mypclose() on pid %d", pid);
 
-    // close the pipe fd
-    // this is required in musl
-    // without it the childs do not exit
-    close(fileno(fp));
+    if (fp) {
+        // close the pipe fd
+        // this is required in musl
+        // without it the childs do not exit
+        close(fileno(fp));
 
-    // close the pipe file pointer
-    fclose(fp);
+        // close the pipe file pointer
+        fclose(fp);
+    }
 
     errno = 0;
 
@@ -285,3 +325,13 @@ int mypclose(FILE *fp, pid_t pid) {
     
     return 0;
 }
+
+int mypclose(FILE *fp, pid_t pid)
+{
+    return custom_pclose(fp, pid);
+}
+
+int netdata_spawn_waitpid(pid_t pid)
+{
+    return custom_pclose(NULL, pid);
+}
\ No newline at end of file
diff --git a/libnetdata/popen/popen.h b/libnetdata/popen/popen.h
index 32f64e460b..f387cff0a1 100644
--- a/libnetdata/popen/popen.h
+++ b/libnetdata/popen/popen.h
@@ -11,6 +11,8 @@
 extern FILE *mypopen(const char *command, volatile pid_t *pidptr);
 extern FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env);
 extern int mypclose(FILE *fp, pid_t pid);
+extern int netdata_spawn(const char *command, volatile pid_t *pidptr);
+extern int netdata_spawn_waitpid(pid_t pid);
 extern void myp_init(void);
 extern void myp_free(void);
 extern int myp_reap(pid_t pid);