Misc SSL improvements 2 (#14334)

* set to wait receive/send when ssl returns wait read/write

* compare the bytes

* set to normal to prevent going into stream mode with incomplete request

* disable wait send

2 files changed, 78 insertions, 47 deletions

diff --git a/web/server/static/static-threaded.c b/web/server/static/static-threaded.c
index aca7d7ec04..7076648c92 100644
--- a/web/server/static/static-threaded.c
+++ b/web/server/static/static-threaded.c
@@ -293,61 +293,73 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) {
     struct web_client *w = (struct web_client *)pi->data;
     int fd = pi->fd;
 
-    if(unlikely(web_client_receive(w) < 0)) {
-        ret = -1;
-        goto cleanup;
-    }
+    web_client_disable_wait_receive(w);
+    web_client_disable_wait_send(w);
 
-    debug(D_WEB_CLIENT, "%llu: processing received data on fd %d.", w->id, fd);
-    worker_is_idle();
-    worker_is_busy(WORKER_JOB_PROCESS);
-    web_client_process_request(w);
+    ssize_t bytes;
+    bytes = web_client_receive(w);
 
-    if (unlikely(w->mode == WEB_CLIENT_MODE_STREAM)) {
-        web_client_send(w);
-    }
+    if (likely(bytes > 0)) {
+        debug(D_WEB_CLIENT, "%llu: processing received data on fd %d.", w->id, fd);
+        worker_is_idle();
+        worker_is_busy(WORKER_JOB_PROCESS);
+        web_client_process_request(w);
 
-    else if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
-        if(w->pollinfo_filecopy_slot == 0) {
-            debug(D_WEB_CLIENT, "%llu: FILECOPY DETECTED ON FD %d", w->id, pi->fd);
-
-            if (unlikely(w->ifd != -1 && w->ifd != w->ofd && w->ifd != fd)) {
-                // add a new socket to poll_events, with the same
-                debug(D_WEB_CLIENT, "%llu: CREATING FILECOPY SLOT ON FD %d", w->id, pi->fd);
-
-                POLLINFO *fpi = poll_add_fd(
-                        pi->p
-                        , w->ifd
-                        , pi->port_acl
-                        , 0
-                        , POLLINFO_FLAG_CLIENT_SOCKET
-                        , "FILENAME"
-                        , ""
-                        , ""
-                        , web_server_file_add_callback
-                        , web_server_file_del_callback
-                        , web_server_file_read_callback
-                        , web_server_file_write_callback
-                        , (void *) w
-                );
-
-                if(fpi)
-                    w->pollinfo_filecopy_slot = fpi->slot;
-                else {
-                    error("Failed to add filecopy fd. Closing client.");
-                    ret = -1;
-                    goto cleanup;
+        if (unlikely(w->mode == WEB_CLIENT_MODE_STREAM)) {
+            web_client_send(w);
+        }
+
+        else if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
+            if(w->pollinfo_filecopy_slot == 0) {
+                debug(D_WEB_CLIENT, "%llu: FILECOPY DETECTED ON FD %d", w->id, pi->fd);
+
+                if (unlikely(w->ifd != -1 && w->ifd != w->ofd && w->ifd != fd)) {
+                    // add a new socket to poll_events, with the same
+                    debug(D_WEB_CLIENT, "%llu: CREATING FILECOPY SLOT ON FD %d", w->id, pi->fd);
+
+                    POLLINFO *fpi = poll_add_fd(
+                                                pi->p
+                                                , w->ifd
+                                                , pi->port_acl
+                                                , 0
+                                                , POLLINFO_FLAG_CLIENT_SOCKET
+                                                , "FILENAME"
+                                                , ""
+                                                , ""
+                                                , web_server_file_add_callback
+                                                , web_server_file_del_callback
+                                                , web_server_file_read_callback
+                                                , web_server_file_write_callback
+                                                , (void *) w
+                                                );
+
+                    if(fpi)
+                        w->pollinfo_filecopy_slot = fpi->slot;
+                    else {
+                        error("Failed to add filecopy fd. Closing client.");
+                        ret = -1;
+                        goto cleanup;
+                    }
                 }
             }
         }
-    }
-    else {
+        else {
+            if(unlikely(w->ifd == fd && web_client_has_wait_receive(w)))
+                *events |= POLLIN;
+        }
+
+        if(unlikely(w->ofd == fd && web_client_has_wait_send(w)))
+            *events |= POLLOUT;
+    } else if(unlikely(bytes < 0)) {
+        ret = -1;
+        goto cleanup;
+    } else if (unlikely(bytes == 0)) {
         if(unlikely(w->ifd == fd && web_client_has_wait_receive(w)))
             *events |= POLLIN;
-    }
 
-    if(unlikely(w->ofd == fd && web_client_has_wait_send(w)))
-        *events |= POLLOUT;
+        if(unlikely(w->ofd == fd && web_client_has_wait_send(w)))
+            *events |= POLLOUT;
+    }
 
     ret = web_server_check_client_status(w);
 
diff --git a/web/server/web_client.c b/web/server/web_client.c
index ffc61c4d8c..61dac5c970 100644
--- a/web/server/web_client.c
+++ b/web/server/web_client.c
@@ -38,6 +38,18 @@ static inline int web_client_crock_socket(struct web_client *w) {
     return 0;
 }
 
+static inline void web_client_enable_wait_from_ssl(struct web_client *w, int bytes) {
+    int ssl_err = SSL_get_error(w->ssl.conn, bytes);
+    if (ssl_err == SSL_ERROR_WANT_READ)
+        web_client_enable_wait_receive(w);
+    else if (ssl_err == SSL_ERROR_WANT_WRITE)
+        web_client_enable_wait_send(w);
+    else if (ssl_err) {
+        web_client_disable_wait_receive(w);
+        web_client_disable_wait_send(w);
+    }
+}
+
 static inline int web_client_uncrock_socket(struct web_client *w) {
 #ifdef TCP_CORK
     if(likely(w->tcp_cork && w->ofd != -1)) {
@@ -1056,6 +1068,7 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
     if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
         if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){
             bytes = netdata_ssl_write(w->ssl.conn, buf, len) ;
+            web_client_enable_wait_from_ssl(w, bytes);
         } else {
             bytes = send(w->ofd,buf, len , flags);
         }
@@ -1211,8 +1224,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
     ssize_t bytes;
 #ifdef ENABLE_HTTPS
     if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
-        if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) )
+        if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) ) {
             bytes = netdata_ssl_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
+            web_client_enable_wait_from_ssl(w, bytes);
+        }
         else {
             while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
                 count++;
@@ -1509,6 +1524,9 @@ void web_client_process_request(struct web_client *w) {
             }
             else {
                 // wait for more data
+                // set to normal to prevent web_server_rcv_callback
+                // from going into stream mode
+                w->mode = WEB_CLIENT_MODE_NORMAL;
                 return;
             }
             break;
@@ -1905,6 +1923,7 @@ ssize_t web_client_receive(struct web_client *w)
     if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
         if ( ( w->ssl.conn ) && (!w->ssl.flags)) {
             bytes = netdata_ssl_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
+            web_client_enable_wait_from_ssl(w, bytes);
         }else {
             bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
         }