Re-write of SSL support in Netdata; restoration of SIGCHLD; detection of stale plugins; streaming improvements (#15113)

* add information about streaming connections to /api/v2/nodes; reset defer time when sender or receivers connect or disconnect

* make each streaming destination respect its SSL settings

* to not send SSL traffic over non-SSL connection

* keep track of outgoing streaming connection attempts

* retry SSL reads when SSL_read() returns SSL_ERROR_WANT_READ

* Revert "retry SSL reads when SSL_read() returns SSL_ERROR_WANT_READ"

This reverts commit 14c858677c6f2d3b08c94f298e2f45ecdb74c801.

* cleanup SSL connections properly

* initialize SSL in rpt before takeover

* sender should free SSL when talking to a non-SSL destination

* do not shutdown SSL when receiver exits

* restore operation of SIGCHLD when the reaper is not enabled

* create an fgets function that checks for data and times out

* work on error handling of plugins exiting

* remove newlines from logs

* global call to waitid(), caching the result for netdata_pclose() to process

* receiver tid

* parser timeouts in 2 minutes instead of 10

* fix crash when UUID is NULL in SQLite

* abstract sqlite3 parsing for uuid and text

* write proper ssl errors on read and write

* fix for SSL_ERROR_WANT_RETRY_VERIFY

* SSL WANT per function

* unified SSL error logging

* fix compilation warning

* additional logging about parser cleanup

* streaming parser should call the pluginsd parser cleanup

* SSL error handling work

* SSL initialization unification

* check for pending data when receiving SSL response with timeout

* macro to check if an SSL connection has been established

* remove SSL_pending()

* check for SSL macros

* use SSL_peek() to find if there is a response

* SSL renames

* more SSL renames & cleanup

* rrdpush ssl connection function

* abstract all SSL functions into security.c

* keep track of SSL connections and always attempt to use SSL read/write when on SSL connection

* signal openssl to skip certificate validation when configured to do so

* better SSL error handling and logging

* SSL code cleanup

* SSL retry on SSL_connect and SSL_accept

* SSL provide default return value for old compilers

* SSL read/write functions emulate system read/write functions

* fix receive/send timeout and switch from SSL_peek() to SSL_pending()

* remove SSL_pending()

* removed sender auto-retry and debug info for initial recevier response

* ssl skip certificate verification config for web server

* ssl errors log ip and port of the peer

* keep ssl with web_client for its whole lifetime

* thread safe socket peers to text

* use error_limit() for common ssl errors

* cleanup

* more cleanup

* coverity fixes

* ssl error logs include both local and remote ip/port info

* remove obsolete code

1 files changed, 28 insertions, 77 deletions

diff --git a/exporting/send_data.c b/exporting/send_data.c
index 045aab6ed6..d91fc50d79 100644
--- a/exporting/send_data.c
+++ b/exporting/send_data.c
@@ -81,37 +81,11 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
     while (*sock != -1 && errno != EWOULDBLOCK) {
         ssize_t r;
 #ifdef ENABLE_HTTPS
-        if (exporting_tls_is_enabled(instance->config.type, options) &&
-            connector_specific_data->conn &&
-            connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
-            r = (ssize_t)SSL_read(connector_specific_data->conn,
-                                  &response->buffer[response->len],
-                                  (int) (response->size - response->len));
-
-            if (likely(r > 0)) {
-                // we received some data
-                response->len += r;
-                stats->received_bytes += r;
-                stats->receptions++;
-                continue;
-            } else {
-                int sslerrno = SSL_get_error(connector_specific_data->conn, (int) r);
-                u_long sslerr = ERR_get_error();
-                char buf[256];
-                switch (sslerrno) {
-                    case SSL_ERROR_WANT_READ:
-                    case SSL_ERROR_WANT_WRITE:
-                        goto endloop;
-                    default:
-                        ERR_error_string_n(sslerr, buf, sizeof(buf));
-                        error("SSL error (%s)",
-                              ERR_error_string((long)SSL_get_error(connector_specific_data->conn, (int)r), NULL));
-                        goto endloop;
-                }
-            }
-        } else {
+        if (SSL_connection(&connector_specific_data->ssl))
+            r = netdata_ssl_read(&connector_specific_data->ssl, &response->buffer[response->len],
+                                 (int) (response->size - response->len));
+        else
             r = recv(*sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
-        }
 #else
         r = recv(*sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
 #endif
@@ -120,11 +94,13 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
             response->len += r;
             stats->received_bytes += r;
             stats->receptions++;
-        } else if (r == 0) {
+        }
+        else if (r == 0) {
             error("EXPORTING: '%s' closed the socket", instance->config.destination);
             close(*sock);
             *sock = -1;
-        } else {
+        }
+        else {
             // failed to receive data
             if (errno != EAGAIN && errno != EWOULDBLOCK) {
                 error("EXPORTING: cannot receive data from '%s'.", instance->config.destination);
@@ -135,9 +111,6 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
         break;
 #endif
     }
-#ifdef ENABLE_HTTPS
-endloop:
-#endif
 
     // if we received data, process them
     if (buffer_strlen(response))
@@ -174,14 +147,16 @@ void simple_connector_send_buffer(
     size_t buffer_len = buffer_strlen(buffer);
 
 #ifdef ENABLE_HTTPS
-    if (exporting_tls_is_enabled(instance->config.type, options) &&
-        connector_specific_data->conn &&
-        connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
+    if (SSL_connection(&connector_specific_data->ssl)) {
+
         if (header_len)
-            header_sent_bytes = (ssize_t)SSL_write(connector_specific_data->conn, buffer_tostring(header), header_len);
+            header_sent_bytes = netdata_ssl_write(&connector_specific_data->ssl, buffer_tostring(header), header_len);
+
         if ((size_t)header_sent_bytes == header_len)
-            buffer_sent_bytes = (ssize_t)SSL_write(connector_specific_data->conn, buffer_tostring(buffer), buffer_len);
-    } else {
+            buffer_sent_bytes = netdata_ssl_write(&connector_specific_data->ssl, buffer_tostring(buffer), buffer_len);
+
+    }
+    else {
         if (header_len)
             header_sent_bytes = send(*sock, buffer_tostring(header), header_len, flags);
         if ((size_t)header_sent_bytes == header_len)
@@ -326,43 +301,19 @@ void simple_connector_worker(void *instance_p)
                     if (sock_delnonblock(sock) < 0)
                         error("Exporting cannot remove the non-blocking flag from socket %d", sock);
 
-                    if (connector_specific_data->conn == NULL) {
-                        connector_specific_data->conn = SSL_new(netdata_ssl_exporting_ctx);
-                        if (connector_specific_data->conn == NULL) {
-                            error("Failed to allocate SSL structure to socket %d.", sock);
-                            connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE;
-                        }
-                    } else {
-                        SSL_clear(connector_specific_data->conn);
-                    }
+                    if(netdata_ssl_open(&connector_specific_data->ssl, netdata_ssl_exporting_ctx, sock)) {
+                        if(netdata_ssl_connect(&connector_specific_data->ssl)) {
+                            info("Exporting established a SSL connection.");
+
+                            struct timeval tv;
+                            tv.tv_sec = timeout.tv_sec / 4;
+                            tv.tv_usec = 0;
+
+                            if (!tv.tv_sec)
+                                tv.tv_sec = 2;
 
-                    if (connector_specific_data->conn) {
-                        if (SSL_set_fd(connector_specific_data->conn, sock) != 1) {
-                            error("Failed to set the socket to the SSL on socket fd %d.", sock);
-                            connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE;
-                        } else {
-                            connector_specific_data->flags = NETDATA_SSL_HANDSHAKE_COMPLETE;
-                            SSL_set_connect_state(connector_specific_data->conn);
-                            int err = SSL_connect(connector_specific_data->conn);
-                            if (err != 1) {
-                                err = SSL_get_error(connector_specific_data->conn, err);
-                                error(
-                                    "SSL cannot connect with the server:  %s ",
-                                    ERR_error_string((long)SSL_get_error(connector_specific_data->conn, err), NULL));
-                                connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE;
-                            } else {
-                                info("Exporting established a SSL connection.");
-
-                                struct timeval tv;
-                                tv.tv_sec = timeout.tv_sec / 4;
-                                tv.tv_usec = 0;
-
-                                if (!tv.tv_sec)
-                                    tv.tv_sec = 2;
-
-                                if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, sizeof(tv)))
-                                    error("Cannot set timeout to socket %d, this can block communication", sock);
-                            }
+                            if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, sizeof(tv)))
+                                error("Cannot set timeout to socket %d, this can block communication", sock);
                         }
                     }
                 }