Re-write of SSL support in Netdata; restoration of SIGCHLD; detection of stale plugins; streaming improvements (#15113)

* add information about streaming connections to /api/v2/nodes; reset defer time when sender or receivers connect or disconnect

* make each streaming destination respect its SSL settings

* to not send SSL traffic over non-SSL connection

* keep track of outgoing streaming connection attempts

* retry SSL reads when SSL_read() returns SSL_ERROR_WANT_READ

* Revert "retry SSL reads when SSL_read() returns SSL_ERROR_WANT_READ"

This reverts commit 14c858677c6f2d3b08c94f298e2f45ecdb74c801.

* cleanup SSL connections properly

* initialize SSL in rpt before takeover

* sender should free SSL when talking to a non-SSL destination

* do not shutdown SSL when receiver exits

* restore operation of SIGCHLD when the reaper is not enabled

* create an fgets function that checks for data and times out

* work on error handling of plugins exiting

* remove newlines from logs

* global call to waitid(), caching the result for netdata_pclose() to process

* receiver tid

* parser timeouts in 2 minutes instead of 10

* fix crash when UUID is NULL in SQLite

* abstract sqlite3 parsing for uuid and text

* write proper ssl errors on read and write

* fix for SSL_ERROR_WANT_RETRY_VERIFY

* SSL WANT per function

* unified SSL error logging

* fix compilation warning

* additional logging about parser cleanup

* streaming parser should call the pluginsd parser cleanup

* SSL error handling work

* SSL initialization unification

* check for pending data when receiving SSL response with timeout

* macro to check if an SSL connection has been established

* remove SSL_pending()

* check for SSL macros

* use SSL_peek() to find if there is a response

* SSL renames

* more SSL renames & cleanup

* rrdpush ssl connection function

* abstract all SSL functions into security.c

* keep track of SSL connections and always attempt to use SSL read/write when on SSL connection

* signal openssl to skip certificate validation when configured to do so

* better SSL error handling and logging

* SSL code cleanup

* SSL retry on SSL_connect and SSL_accept

* SSL provide default return value for old compilers

* SSL read/write functions emulate system read/write functions

* fix receive/send timeout and switch from SSL_peek() to SSL_pending()

* remove SSL_pending()

* removed sender auto-retry and debug info for initial recevier response

* ssl skip certificate verification config for web server

* ssl errors log ip and port of the peer

* keep ssl with web_client for its whole lifetime

* thread safe socket peers to text

* use error_limit() for common ssl errors

* cleanup

* more cleanup

* coverity fixes

* ssl error logs include both local and remote ip/port info

* remove obsolete code

5 files changed, 66 insertions, 136 deletions

diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 551fee4c3a..a0347f6fe9 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -3291,7 +3291,6 @@ static void rrd2rrdr_convert_values_to_percentage_of_total(RRDR *r) {
 
 static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) {
     QUERY_TARGET *qt = r_tmp->internal.qt;
-    RRDR_OPTIONS options = qt->window.options;
 
     if(!r_tmp->group_by.r) {
         // v1 query
diff --git a/web/server/static/static-threaded.c b/web/server/static/static-threaded.c
index 52bb56cd61..4cb3dcd923 100644
--- a/web/server/static/static-threaded.c
+++ b/web/server/static/static-threaded.c
@@ -211,58 +211,32 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data
     }
 
 #ifdef ENABLE_HTTPS
-    if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
-        if( sock_delnonblock(w->ifd) < 0 ){
-            error("Web server cannot remove the non-blocking flag from socket %d",w->ifd);
-        }
+    if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) {
+        sock_delnonblock(w->ifd);
 
         //Read the first 7 bytes from the message, but the message
         //is not removed from the queue, because we are using MSG_PEEK
         char test[8];
-        if ( recv(w->ifd,test, 7,MSG_PEEK) == 7 ) {
-            test[7] = 0x00;
+        if ( recv(w->ifd,test, 7, MSG_PEEK) == 7 ) {
+            test[7] = '\0';
         }
         else {
-            //Case I do not have success to read 7 bytes,
-            //this means that the mensage was not completely read, so
-            //I cannot identify it yet.
+            // we couldn't read 7 bytes
             sock_setnonblock(w->ifd);
             goto cleanup;
         }
 
-        //The next two ifs are not together because I am reusing SSL structure
-        if (!w->ssl.conn)
-        {
-            w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
-            if ( w->ssl.conn ) {
-                SSL_set_accept_state(w->ssl.conn);
-            } else {
-                error("Failed to create SSL context on socket fd %d.", w->ifd);
-                if (test[0] < 0x18){
-                    WEB_CLIENT_IS_DEAD(w);
-                    sock_setnonblock(w->ifd);
-                    goto cleanup;
-                }
-            }
+        if(test[0] > 0x17) {
+            // no SSL
+            netdata_ssl_close(&w->ssl); // free any previous SSL data
         }
-
-        if (w->ssl.conn) {
-            if (SSL_set_fd(w->ssl.conn, w->ifd) != 1) {
-                error("Failed to set the socket to the SSL on socket fd %d.", w->ifd);
-                //The client is not set dead, because I received a normal HTTP request
-                //instead a Client Hello(HTTPS).
-                if ( test[0] < 0x18 ){
-                    WEB_CLIENT_IS_DEAD(w);
-                }
-            }
-            else{
-                w->ssl.flags = security_process_accept(w->ssl.conn, (int)test[0]);
-            }
+        else {
+            // SSL
+            if(!netdata_ssl_open(&w->ssl, netdata_ssl_web_server_ctx, w->ifd) || !netdata_ssl_accept(&w->ssl))
+                WEB_CLIENT_IS_DEAD(w);
         }
 
         sock_setnonblock(w->ifd);
-    } else{
-        w->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
     }
 #endif
 
@@ -525,9 +499,15 @@ void *socket_listen_main_static_threaded(void *ptr) {
     if(!api_sockets.opened)
         fatal("LISTENER: no listen sockets available.");
 
+    netdata_ssl_validate_certificate = !config_get_boolean(CONFIG_SECTION_WEB, "ssl skip certificate verification", !netdata_ssl_validate_certificate);
+
+    if(!netdata_ssl_validate_certificate_sender)
+        info("SSL: web server will skip SSL certificates verification.");
+
 #ifdef ENABLE_HTTPS
-    security_start_ssl(NETDATA_SSL_CONTEXT_SERVER);
+    netdata_ssl_initialize_ctx(NETDATA_SSL_WEB_SERVER_CTX);
 #endif
+
     // 6 threads is the optimal value
     // since 6 are the parallel connections browsers will do
     // so, if the machine has more CPUs, avoid using resources unnecessarily
@@ -541,6 +521,7 @@ void *socket_listen_main_static_threaded(void *ptr) {
     static_threaded_workers_count = config_get_number(CONFIG_SECTION_WEB, "web server threads", def_thread_count);
 
     if (static_threaded_workers_count < 1) static_threaded_workers_count = 1;
+
 #ifdef ENABLE_HTTPS
     // See https://github.com/netdata/netdata/issues/11081#issuecomment-831998240 for more details
     if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) {
diff --git a/web/server/web_client.c b/web/server/web_client.c
index 318d4ebcf8..6e3c1225ea 100644
--- a/web/server/web_client.c
+++ b/web/server/web_client.c
@@ -34,11 +34,10 @@ static inline int web_client_crock_socket(struct web_client *w __maybe_unused) {
     return 0;
 }
 
-static inline void web_client_enable_wait_from_ssl(struct web_client *w, int bytes) {
-    int ssl_err = SSL_get_error(w->ssl.conn, bytes);
-    if (ssl_err == SSL_ERROR_WANT_READ)
+static inline void web_client_enable_wait_from_ssl(struct web_client *w) {
+    if (w->ssl.ssl_errno == SSL_ERROR_WANT_READ)
         web_client_enable_ssl_wait_receive(w);
-    else if (ssl_err == SSL_ERROR_WANT_WRITE)
+    else if (w->ssl.ssl_errno == SSL_ERROR_WANT_WRITE)
         web_client_enable_ssl_wait_send(w);
     else {
         web_client_disable_ssl_wait_receive(w);
@@ -99,15 +98,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
         freez(w->post_payload);
         w->post_payload = NULL;
         w->post_payload_size = 0;
-
-#ifdef ENABLE_HTTPS
-        if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
-            if (w->ssl.conn) {
-                SSL_free(w->ssl.conn);
-                w->ssl.conn = NULL;
-            }
-        }
-#endif
     }
     else {
         // the web client is to be re-used
@@ -121,7 +111,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
         buffer_reset(w->response.data);
 
         // leave w->post_payload
-        // leave w->ssl
     }
 
     freez(w->server_host);
@@ -794,12 +783,10 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
             //  web_client_enable_deflate(w, 0);
         }
     }
-#ifdef ENABLE_HTTPS
     else if(hash == hash_forwarded_proto && !strcasecmp(s, "X-Forwarded-Proto")) {
         if(strcasestr(v, "https"))
-            w->ssl.flags |= NETDATA_SSL_PROXY_HTTPS;
+            w->flags |= WEB_CLIENT_FLAG_PROXY_HTTPS;
     }
-#endif
     else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")) {
         char buffer[NI_MAXHOST];
         strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
@@ -839,7 +826,7 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
         s = &s[7];
 
 #ifdef ENABLE_HTTPS
-        if (w->ssl.flags && web_client_is_using_ssl_force(w)){
+        if (!SSL_connection(&w->ssl) && web_client_is_using_ssl_force(w)) {
             w->header_parse_tries = 0;
             w->header_parse_last_size = 0;
             web_client_disable_wait_receive(w);
@@ -980,8 +967,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
                 *ue = c;
 
 #ifdef ENABLE_HTTPS
-                if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
-                    if ((w->ssl.conn) && ((w->ssl.flags & NETDATA_SSL_NO_HANDSHAKE) && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM))  ) {
+                if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
+                    if (!w->ssl.conn && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) {
                         w->header_parse_tries = 0;
                         w->header_parse_last_size = 0;
                         web_client_disable_wait_receive(w);
@@ -1010,16 +997,15 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
 {
     ssize_t bytes;
 #ifdef ENABLE_HTTPS
-    if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
-        if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){
-            bytes = netdata_ssl_write(w->ssl.conn, buf, len) ;
-            web_client_enable_wait_from_ssl(w, bytes);
-        } else {
-            bytes = send(w->ofd,buf, len , flags);
+    if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) {
+        if (SSL_connection(&w->ssl)) {
+            bytes = netdata_ssl_write(&w->ssl, buf, len) ;
+            web_client_enable_wait_from_ssl(w);
         }
-    } else {
+        else
+            bytes = send(w->ofd,buf, len , flags);
+    } else
         bytes = send(w->ofd,buf, len , flags);
-    }
 #else
     bytes = send(w->ofd, buf, len, flags);
 #endif
@@ -1156,10 +1142,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
     size_t count = 0;
     ssize_t bytes;
 #ifdef ENABLE_HTTPS
-    if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
-        if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) ) {
-            bytes = netdata_ssl_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
-            web_client_enable_wait_from_ssl(w, bytes);
+    if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
+        if (SSL_connection(&w->ssl)) {
+            bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
+            web_client_enable_wait_from_ssl(w);
         }
         else {
             while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
@@ -1260,11 +1246,11 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
             if(!url) { //no delim found
                 debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id);
                 char *protocol, *url_host;
+                protocol = (
 #ifdef ENABLE_HTTPS
-                protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http";
-#else
-                protocol = "http";
+                        SSL_connection(&w->ssl) ||
 #endif
+                        (w->flags & WEB_CLIENT_FLAG_PROXY_HTTPS)) ? "https" : "http";
 
                 url_host = w->forwarded_host;
                 if(!url_host) {
@@ -1948,11 +1934,12 @@ ssize_t web_client_receive(struct web_client *w)
     buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
 
 #ifdef ENABLE_HTTPS
-    if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
-        if ( ( w->ssl.conn ) && (!w->ssl.flags)) {
-            bytes = netdata_ssl_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
-            web_client_enable_wait_from_ssl(w, bytes);
-        }else {
+    if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
+        if (SSL_connection(&w->ssl)) {
+            bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
+            web_client_enable_wait_from_ssl(w);
+        }
+        else {
             bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
         }
     }
@@ -1985,26 +1972,6 @@ ssize_t web_client_receive(struct web_client *w)
 }
 
 
-int web_client_socket_is_now_used_for_streaming(struct web_client *w) {
-    // prevent the web_client from closing the streaming socket
-
-    WEB_CLIENT_IS_DEAD(w);
-
-    if(web_server_mode == WEB_SERVER_MODE_STATIC_THREADED) {
-        web_client_flag_set(w, WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET);
-    }
-    else {
-        if(w->ifd == w->ofd)
-            w->ifd = w->ofd = -1;
-        else
-            w->ifd = -1;
-    }
-
-    buffer_flush(w->response.data);
-
-    return HTTP_RESP_OK;
-}
-
 void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string) {
     char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2];
     buffer[0] = '\0';
@@ -2052,25 +2019,6 @@ void web_client_decode_path_and_query_string(struct web_client *w, const char *p
     }
 }
 
-#ifdef ENABLE_HTTPS
-void web_client_reuse_ssl(struct web_client *w) {
-    if (netdata_ssl_srv_ctx) {
-        if (w->ssl.conn) {
-            SSL_SESSION *session = SSL_get_session(w->ssl.conn);
-            SSL *old = w->ssl.conn;
-            w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
-            if (session) {
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111
-                if (SSL_SESSION_is_resumable(session))
-#endif
-                    SSL_set_session(w->ssl.conn, session);
-            }
-            SSL_free(old);
-        }
-    }
-}
-#endif
-
 void web_client_zero(struct web_client *w) {
     // zero everything about it - but keep the buffers
 
@@ -2085,8 +2033,7 @@ void web_client_zero(struct web_client *w) {
     BUFFER *b6 = w->url_query_string_decoded;
 
 #ifdef ENABLE_HTTPS
-    web_client_reuse_ssl(w);
-    SSL *ssl = w->ssl.conn;
+    NETDATA_SSL ssl = w->ssl;
 #endif
 
     size_t use_count = w->use_count;
@@ -2100,9 +2047,7 @@ void web_client_zero(struct web_client *w) {
     w->use_count = use_count;
 
 #ifdef ENABLE_HTTPS
-    w->ssl.conn = ssl;
-    w->ssl.flags = NETDATA_SSL_START;
-    debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
+    w->ssl = ssl;
 #endif
 
     // restore the pointers of the buffers
@@ -2116,6 +2061,11 @@ void web_client_zero(struct web_client *w) {
 
 struct web_client *web_client_create(size_t *statistics_memory_accounting) {
     struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
+
+#ifdef ENABLE_HTTPS
+    w->ssl = NETDATA_SSL_UNSET_CONNECTION;
+#endif
+
     w->use_count = 1;
     w->statistics.memory_accounting = statistics_memory_accounting;
 
@@ -2132,6 +2082,10 @@ struct web_client *web_client_create(size_t *statistics_memory_accounting) {
 }
 
 void web_client_free(struct web_client *w) {
+#ifdef ENABLE_HTTPS
+    netdata_ssl_close(&w->ssl);
+#endif
+
     web_client_reset_allocations(w, true);
 
     __atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
diff --git a/web/server/web_client.h b/web/server/web_client.h
index dcb127a322..4c2b06a70e 100644
--- a/web/server/web_client.h
+++ b/web/server/web_client.h
@@ -52,6 +52,8 @@ typedef enum web_client_flags {
 
     WEB_CLIENT_FLAG_SSL_WAIT_RECEIVE = 1 << 11, // if set, we are waiting more input data from an ssl conn
     WEB_CLIENT_FLAG_SSL_WAIT_SEND = 1 << 12,    // if set, we have data to send to the client from an ssl conn
+
+    WEB_CLIENT_FLAG_PROXY_HTTPS = 1 << 13, // if set, the client reaches us via an https proxy
 } WEB_CLIENT_FLAGS;
 
 #define web_client_flag_check(w, flag) ((w)->flags & (flag))
@@ -168,7 +170,7 @@ struct web_client {
     size_t pollinfo_filecopy_slot;      // POLLINFO slot of the file read
 
 #ifdef ENABLE_HTTPS
-    struct netdata_ssl ssl;
+    NETDATA_SSL ssl;
 #endif
 
     struct {                            // A callback to check if the query should be interrupted / stopped
@@ -213,16 +215,10 @@ int mysendfile(struct web_client *w, char *filename);
 void web_client_build_http_header(struct web_client *w);
 char *strip_control_characters(char *url);
 
-int web_client_socket_is_now_used_for_streaming(struct web_client *w);
-
 void web_client_zero(struct web_client *w);
 struct web_client *web_client_create(size_t *statistics_memory_accounting);
 void web_client_free(struct web_client *w);
 
-#ifdef ENABLE_HTTPS
-void web_client_reuse_ssl(struct web_client *w);
-#endif
-
 #include "web/api/web_api_v1.h"
 #include "web/api/web_api_v2.h"
 #include "daemon/common.h"
diff --git a/web/server/web_client_cache.c b/web/server/web_client_cache.c
index b410ba7f90..394bea32b4 100644
--- a/web/server/web_client_cache.c
+++ b/web/server/web_client_cache.c
@@ -104,11 +104,6 @@ struct web_client *web_client_get_from_cache(void) {
         // allocate it
         w = web_client_create(&netdata_buffers_statistics.buffers_web);
 
-#ifdef ENABLE_HTTPS
-        w->ssl.flags = NETDATA_SSL_START;
-        debug(D_WEB_CLIENT_ACCESS,"Starting SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
-#endif
-
         netdata_spinlock_lock(&web_clients_cache.used.spinlock);
         web_clients_cache.used.allocated++;
     }
@@ -127,6 +122,11 @@ struct web_client *web_client_get_from_cache(void) {
 }
 
 void web_client_release_to_cache(struct web_client *w) {
+
+#ifdef ENABLE_HTTPS
+    netdata_ssl_close(&w->ssl);
+#endif
+
     // unlink it from the used
     netdata_spinlock_lock(&web_clients_cache.used.spinlock);
     DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);