Refactor the async wait fd logic

Implementation experience has shown that the original plan for async wait
fds was too simplistic. Originally the async logic created a pipe internally
and user/engine code could then get access to it via API calls. It is more
flexible if the engine is able to create its own fd and provide it to the
async code.

Another issue is that there can be a lot of churn in the fd value within
the context of (say) a single SSL connection leading to continually adding
and removing fds from (say) epoll. It is better if we can provide some
stability of the fd value across a whole SSL connection. This is
problematic because an engine has no concept of an SSL connection.

This commit refactors things to introduce an ASYNC_WAIT_CTX which acts as a
proxy for an SSL connection down at the engine layer.

Reviewed-by: Richard Levitte <levitte@openssl.org>

8 files changed, 266 insertions, 137 deletions

diff --git a/crypto/async/Makefile.in b/crypto/async/Makefile.in
index 277aa9ec32..97b5385c08 100644
--- a/crypto/async/Makefile.in
+++ b/crypto/async/Makefile.in
@@ -17,8 +17,8 @@ TEST=
 APPS=
 
 LIB=$(TOP)/libcrypto.a
-LIBSRC=async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
-LIBOBJ=async.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
+LIBSRC=async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+LIBOBJ=async.o async_wait.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
 
 SRC= $(LIBSRC)
 
diff --git a/crypto/async/arch/async_null.c b/crypto/async/arch/async_null.c
index 2b1d28eaf3..03f8ebfa10 100644
--- a/crypto/async/arch/async_null.c
+++ b/crypto/async/arch/async_null.c
@@ -55,26 +55,6 @@
 
 #ifdef ASYNC_NULL
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    return -1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    return 0;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    return -1;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    return -1;
-}
-
 int async_global_init(void)
 {
     return 0;
diff --git a/crypto/async/arch/async_posix.c b/crypto/async/arch/async_posix.c
index 57cce7b4c4..626471d370 100644
--- a/crypto/async/arch/async_posix.c
+++ b/crypto/async/arch/async_posix.c
@@ -103,36 +103,4 @@ void async_fibre_free(async_fibre *fibre)
     fibre->fibre.uc_stack.ss_sp = NULL;
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (pipe(pipefds) == 0)
-        return 1;
-
-    return 0;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (close(fd) != 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    if (write(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    if (read(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
 #endif
diff --git a/crypto/async/arch/async_win.c b/crypto/async/arch/async_win.c
index e862e2548f..c0776b1239 100644
--- a/crypto/async/arch/async_win.c
+++ b/crypto/async/arch/async_win.c
@@ -127,42 +127,6 @@ VOID CALLBACK async_start_func_win(PVOID unused)
     async_start_func();
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (CreatePipe(&pipefds[0], &pipefds[1], NULL, 256) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (CloseHandle(fd) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    DWORD numwritten = 0;
-
-    if (WriteFile(fd, buf, 1, &numwritten, NULL) && numwritten == 1)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    DWORD numread = 0;
-
-    if (ReadFile(fd, buf, 1, &numread, NULL) && numread == 1)
-        return 1;
-
-    return 0;
-}
-
 async_pool *async_get_pool(void)
 {
     return (async_pool *)TlsGetValue(asyncwinpool);
diff --git a/crypto/async/async.c b/crypto/async/async.c
index ebc5ebbe93..de2ec4a7c6 100644
--- a/crypto/async/async.c
+++ b/crypto/async/async.c
@@ -119,26 +119,14 @@ static int async_ctx_free(void)
 static ASYNC_JOB *async_job_new(void)
 {
     ASYNC_JOB *job = NULL;
-    OSSL_ASYNC_FD pipefds[2];
 
-    job = OPENSSL_malloc(sizeof (ASYNC_JOB));
+    job = OPENSSL_zalloc(sizeof (ASYNC_JOB));
     if (job == NULL) {
         ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
 
-    if (!async_pipe(pipefds)) {
-        OPENSSL_free(job);
-        ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ASYNC_R_CANNOT_CREATE_WAIT_PIPE);
-        return NULL;
-    }
-
-    job->wake_set = 0;
-    job->wait_fd = pipefds[0];
-    job->wake_fd = pipefds[1];
-
     job->status = ASYNC_JOB_RUNNING;
-    job->funcargs = NULL;
 
     return job;
 }
@@ -148,8 +136,6 @@ static void async_job_free(ASYNC_JOB *job)
     if (job != NULL) {
         OPENSSL_free(job->funcargs);
         async_fibre_free(&job->fibrectx);
-        async_close_fd(job->wait_fd);
-        async_close_fd(job->wake_fd);
         OPENSSL_free(job);
     }
 }
@@ -219,8 +205,8 @@ void async_start_func(void)
     }
 }
 
-int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                         void *args, size_t size)
+int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret,
+                    int (*func)(void *), void *args, size_t size)
 {
     async_ctx *ctx = async_get_ctx();
     if (ctx == NULL)
@@ -237,6 +223,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         if (ctx->currjob != NULL) {
             if (ctx->currjob->status == ASYNC_JOB_STOPPING) {
                 *ret = ctx->currjob->ret;
+                ctx->currjob->waitctx = NULL;
                 async_release_job(ctx->currjob);
                 ctx->currjob = NULL;
                 *job = NULL;
@@ -289,6 +276,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         }
 
         ctx->currjob->func = func;
+        ctx->currjob->waitctx = wctx;
         if (!async_fibre_swapcontext(&ctx->dispatcher,
                 &ctx->currjob->fibrectx, 1)) {
             ASYNCerr(ASYNC_F_ASYNC_START_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
@@ -303,7 +291,6 @@ err:
     return ASYNC_ERR;
 }
 
-
 int ASYNC_pause_job(void)
 {
     ASYNC_JOB *job;
@@ -327,6 +314,8 @@ int ASYNC_pause_job(void)
         ASYNCerr(ASYNC_F_ASYNC_PAUSE_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
         return 0;
     }
+    /* Reset counts of added and deleted fds */
+    async_wait_ctx_reset_counts(job->waitctx);
 
     return 1;
 }
@@ -441,28 +430,9 @@ ASYNC_JOB *ASYNC_get_current_job(void)
     return ctx->currjob;
 }
 
-OSSL_ASYNC_FD ASYNC_get_wait_fd(ASYNC_JOB *job)
-{
-    return job->wait_fd;
-}
-
-void ASYNC_wake(ASYNC_JOB *job)
-{
-    char dummy = 0;
-
-    if (job->wake_set)
-        return;
-    async_write1(job->wake_fd, &dummy);
-    job->wake_set = 1;
-}
-
-void ASYNC_clear_wake(ASYNC_JOB *job)
+ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job)
 {
-    char dummy = 0;
-    if (!job->wake_set)
-        return;
-    async_read1(job->wait_fd, &dummy);
-    job->wake_set = 0;
+    return job->waitctx;
 }
 
 void ASYNC_block_pause(void)
diff --git a/crypto/async/async_locl.h b/crypto/async/async_locl.h
index 53a192b329..4caf16db2c 100644
--- a/crypto/async/async_locl.h
+++ b/crypto/async/async_locl.h
@@ -81,9 +81,23 @@ struct async_job_st {
     void *funcargs;
     int ret;
     int status;
-    int wake_set;
-    OSSL_ASYNC_FD wait_fd;
-    OSSL_ASYNC_FD wake_fd;
+    ASYNC_WAIT_CTX *waitctx;
+};
+
+struct fd_lookup_st {
+    const void *key;
+    OSSL_ASYNC_FD fd;
+    void *custom_data;
+    void (*cleanup)(ASYNC_WAIT_CTX *, const void *, OSSL_ASYNC_FD, void *);
+    int add;
+    int del;
+    struct fd_lookup_st *next;
+};
+
+struct async_wait_ctx_st {
+    struct fd_lookup_st *fds;
+    size_t numadd;
+    size_t numdel;
 };
 
 DEFINE_STACK_OF(ASYNC_JOB)
@@ -98,7 +112,6 @@ int async_global_init(void);
 void async_local_cleanup(void);
 void async_global_cleanup(void);
 void async_start_func(void);
-int async_pipe(OSSL_ASYNC_FD *pipefds);
-int async_close_fd(OSSL_ASYNC_FD fd);
-int async_write1(OSSL_ASYNC_FD fd, const void *buf);
-int async_read1(OSSL_ASYNC_FD fd, void *buf);
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx);
+
diff --git a/crypto/async/async_wait.c b/crypto/async/async_wait.c
new file mode 100644
index 0000000000..9d90e524c3
--- /dev/null
+++ b/crypto/async/async_wait.c
@@ -0,0 +1,233 @@
+/*
+ * Written by Matt Caswell (matt@openssl.org) for the OpenSSL project.
+ */
+/* ====================================================================
+ * Copyright (c) 2016 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    licensing@OpenSSL.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ */
+
+/* This must be the first #include file */
+#include "async_locl.h"
+
+#include <openssl/err.h>
+
+ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void)
+{
+    return OPENSSL_zalloc(sizeof(ASYNC_WAIT_CTX));
+}
+
+void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr;
+
+    if (ctx == NULL)
+        return;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has already been deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->cleanup != NULL)
+            curr->cleanup(ctx, curr->key, curr->fd, curr->custom_data);
+        curr = curr->next;
+    }
+
+    OPENSSL_free(ctx);
+}
+int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                               OSSL_ASYNC_FD fd, void *custom_data,
+                               void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *))
+{
+    struct fd_lookup_st *fdlookup;
+
+    fdlookup = OPENSSL_zalloc(sizeof *fdlookup);
+    if (fdlookup == NULL)
+        return 0;
+
+    fdlookup->key = key;
+    fdlookup->fd = fd;
+    fdlookup->custom_data = custom_data;
+    fdlookup->cleanup = cleanup;
+    fdlookup->add = 1;
+    fdlookup->next = ctx->fds;
+    ctx->fds = fdlookup;
+    ctx->numadd++;
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                          OSSL_ASYNC_FD *fd, void **custom_data)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            *fd = curr->fd;
+            *custom_data = curr->custom_data;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                               size_t *numfds)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    *numfds = 0;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (fd != NULL) {
+            *fd = curr->fd;
+            fd++;
+        }
+        (*numfds)++;
+        curr = curr->next;
+    }
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                   size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                   size_t *numdelfds)
+{
+    struct fd_lookup_st *curr;
+
+    *numaddfds = ctx->numadd;
+    *numdelfds = ctx->numdel;
+    if (addfd == NULL && delfd == NULL)
+        return 1;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        /* We ignore fds that have been marked as both added and deleted */
+        if (curr->del && !curr->add && (delfd != NULL)) {
+            *delfd = curr->fd;
+            delfd++;
+        }
+        if (curr->add && !curr->del && (addfd != NULL)) {
+            *addfd = curr->fd;
+            addfd++;
+        }
+        curr = curr->next;
+    }
+
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted already so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            /*
+             * Mark it as deleted. We don't call cleanup if explicitly asked
+             * to clear an fd. We assume the caller is going to do that
+             */
+            curr->del = 1;
+            ctx->numdel++;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr, *prev = NULL;
+
+    ctx->numadd = 0;
+    ctx->numdel = 0;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        if (curr->del) {
+            if (prev == NULL)
+                ctx->fds = curr->next;
+            else
+                prev->next = curr->next;
+            OPENSSL_free(curr);
+            if (prev == NULL)
+                curr = ctx->fds;
+            else
+                curr = prev->next;
+            continue;
+        }
+        if (curr->add) {
+            curr->add = 0;
+        }
+        prev = curr;
+        curr = curr->next;
+    }
+}
diff --git a/crypto/async/build.info b/crypto/async/build.info
index d975003888..278e3e9f89 100644
--- a/crypto/async/build.info
+++ b/crypto/async/build.info
@@ -1,3 +1,4 @@
 LIBS=../../libcrypto
 SOURCE[../../libcrypto]=\
-        async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+        async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c \
+        arch/async_null.c