21 files changed, 792 insertions, 325 deletions

diff --git a/apps/apps.c b/apps/apps.c
index 5ad090a44c..9f60e76dd8 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -2638,15 +2638,27 @@ BIO *bio_open_default_quiet(const char *filename, char mode, int format)
 
 void wait_for_async(SSL *s)
 {
-    int width, fd;
+    int width = 0;
     fd_set asyncfds;
+    OSSL_ASYNC_FD *fds;
+    size_t numfds;
 
-    fd = SSL_get_async_wait_fd(s);
-    if (fd < 0)
+    if (!SSL_get_all_async_fds(s, NULL, &numfds))
+        return;
+    if (numfds == 0)
         return;
+    fds = OPENSSL_malloc(sizeof(OSSL_ASYNC_FD) * numfds);
+    if (!SSL_get_all_async_fds(s, fds, &numfds)) {
+        OPENSSL_free(fds);
+    }
 
-    width = fd + 1;
     FD_ZERO(&asyncfds);
-    openssl_fdset(fd, &asyncfds);
+    while (numfds > 0) {
+        if (width <= (int)*fds)
+            width = (int)*fds + 1;
+        openssl_fdset((int)*fds, &asyncfds);
+        numfds--;
+        fds++;
+    }
     select(width, (void *)&asyncfds, NULL, NULL, NULL);
 }
diff --git a/crypto/async/Makefile.in b/crypto/async/Makefile.in
index 277aa9ec32..97b5385c08 100644
--- a/crypto/async/Makefile.in
+++ b/crypto/async/Makefile.in
@@ -17,8 +17,8 @@ TEST=
 APPS=
 
 LIB=$(TOP)/libcrypto.a
-LIBSRC=async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
-LIBOBJ=async.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
+LIBSRC=async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+LIBOBJ=async.o async_wait.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
 
 SRC= $(LIBSRC)
 
diff --git a/crypto/async/arch/async_null.c b/crypto/async/arch/async_null.c
index 2b1d28eaf3..03f8ebfa10 100644
--- a/crypto/async/arch/async_null.c
+++ b/crypto/async/arch/async_null.c
@@ -55,26 +55,6 @@
 
 #ifdef ASYNC_NULL
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    return -1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    return 0;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    return -1;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    return -1;
-}
-
 int async_global_init(void)
 {
     return 0;
diff --git a/crypto/async/arch/async_posix.c b/crypto/async/arch/async_posix.c
index 57cce7b4c4..626471d370 100644
--- a/crypto/async/arch/async_posix.c
+++ b/crypto/async/arch/async_posix.c
@@ -103,36 +103,4 @@ void async_fibre_free(async_fibre *fibre)
     fibre->fibre.uc_stack.ss_sp = NULL;
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (pipe(pipefds) == 0)
-        return 1;
-
-    return 0;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (close(fd) != 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    if (write(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    if (read(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
 #endif
diff --git a/crypto/async/arch/async_win.c b/crypto/async/arch/async_win.c
index e862e2548f..c0776b1239 100644
--- a/crypto/async/arch/async_win.c
+++ b/crypto/async/arch/async_win.c
@@ -127,42 +127,6 @@ VOID CALLBACK async_start_func_win(PVOID unused)
     async_start_func();
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (CreatePipe(&pipefds[0], &pipefds[1], NULL, 256) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (CloseHandle(fd) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    DWORD numwritten = 0;
-
-    if (WriteFile(fd, buf, 1, &numwritten, NULL) && numwritten == 1)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    DWORD numread = 0;
-
-    if (ReadFile(fd, buf, 1, &numread, NULL) && numread == 1)
-        return 1;
-
-    return 0;
-}
-
 async_pool *async_get_pool(void)
 {
     return (async_pool *)TlsGetValue(asyncwinpool);
diff --git a/crypto/async/async.c b/crypto/async/async.c
index ebc5ebbe93..de2ec4a7c6 100644
--- a/crypto/async/async.c
+++ b/crypto/async/async.c
@@ -119,26 +119,14 @@ static int async_ctx_free(void)
 static ASYNC_JOB *async_job_new(void)
 {
     ASYNC_JOB *job = NULL;
-    OSSL_ASYNC_FD pipefds[2];
 
-    job = OPENSSL_malloc(sizeof (ASYNC_JOB));
+    job = OPENSSL_zalloc(sizeof (ASYNC_JOB));
     if (job == NULL) {
         ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
 
-    if (!async_pipe(pipefds)) {
-        OPENSSL_free(job);
-        ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ASYNC_R_CANNOT_CREATE_WAIT_PIPE);
-        return NULL;
-    }
-
-    job->wake_set = 0;
-    job->wait_fd = pipefds[0];
-    job->wake_fd = pipefds[1];
-
     job->status = ASYNC_JOB_RUNNING;
-    job->funcargs = NULL;
 
     return job;
 }
@@ -148,8 +136,6 @@ static void async_job_free(ASYNC_JOB *job)
     if (job != NULL) {
         OPENSSL_free(job->funcargs);
         async_fibre_free(&job->fibrectx);
-        async_close_fd(job->wait_fd);
-        async_close_fd(job->wake_fd);
         OPENSSL_free(job);
     }
 }
@@ -219,8 +205,8 @@ void async_start_func(void)
     }
 }
 
-int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                         void *args, size_t size)
+int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret,
+                    int (*func)(void *), void *args, size_t size)
 {
     async_ctx *ctx = async_get_ctx();
     if (ctx == NULL)
@@ -237,6 +223,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         if (ctx->currjob != NULL) {
             if (ctx->currjob->status == ASYNC_JOB_STOPPING) {
                 *ret = ctx->currjob->ret;
+                ctx->currjob->waitctx = NULL;
                 async_release_job(ctx->currjob);
                 ctx->currjob = NULL;
                 *job = NULL;
@@ -289,6 +276,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         }
 
         ctx->currjob->func = func;
+        ctx->currjob->waitctx = wctx;
         if (!async_fibre_swapcontext(&ctx->dispatcher,
                 &ctx->currjob->fibrectx, 1)) {
             ASYNCerr(ASYNC_F_ASYNC_START_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
@@ -303,7 +291,6 @@ err:
     return ASYNC_ERR;
 }
 
-
 int ASYNC_pause_job(void)
 {
     ASYNC_JOB *job;
@@ -327,6 +314,8 @@ int ASYNC_pause_job(void)
         ASYNCerr(ASYNC_F_ASYNC_PAUSE_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
         return 0;
     }
+    /* Reset counts of added and deleted fds */
+    async_wait_ctx_reset_counts(job->waitctx);
 
     return 1;
 }
@@ -441,28 +430,9 @@ ASYNC_JOB *ASYNC_get_current_job(void)
     return ctx->currjob;
 }
 
-OSSL_ASYNC_FD ASYNC_get_wait_fd(ASYNC_JOB *job)
-{
-    return job->wait_fd;
-}
-
-void ASYNC_wake(ASYNC_JOB *job)
-{
-    char dummy = 0;
-
-    if (job->wake_set)
-        return;
-    async_write1(job->wake_fd, &dummy);
-    job->wake_set = 1;
-}
-
-void ASYNC_clear_wake(ASYNC_JOB *job)
+ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job)
 {
-    char dummy = 0;
-    if (!job->wake_set)
-        return;
-    async_read1(job->wait_fd, &dummy);
-    job->wake_set = 0;
+    return job->waitctx;
 }
 
 void ASYNC_block_pause(void)
diff --git a/crypto/async/async_locl.h b/crypto/async/async_locl.h
index 53a192b329..4caf16db2c 100644
--- a/crypto/async/async_locl.h
+++ b/crypto/async/async_locl.h
@@ -81,9 +81,23 @@ struct async_job_st {
     void *funcargs;
     int ret;
     int status;
-    int wake_set;
-    OSSL_ASYNC_FD wait_fd;
-    OSSL_ASYNC_FD wake_fd;
+    ASYNC_WAIT_CTX *waitctx;
+};
+
+struct fd_lookup_st {
+    const void *key;
+    OSSL_ASYNC_FD fd;
+    void *custom_data;
+    void (*cleanup)(ASYNC_WAIT_CTX *, const void *, OSSL_ASYNC_FD, void *);
+    int add;
+    int del;
+    struct fd_lookup_st *next;
+};
+
+struct async_wait_ctx_st {
+    struct fd_lookup_st *fds;
+    size_t numadd;
+    size_t numdel;
 };
 
 DEFINE_STACK_OF(ASYNC_JOB)
@@ -98,7 +112,6 @@ int async_global_init(void);
 void async_local_cleanup(void);
 void async_global_cleanup(void);
 void async_start_func(void);
-int async_pipe(OSSL_ASYNC_FD *pipefds);
-int async_close_fd(OSSL_ASYNC_FD fd);
-int async_write1(OSSL_ASYNC_FD fd, const void *buf);
-int async_read1(OSSL_ASYNC_FD fd, void *buf);
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx);
+
diff --git a/crypto/async/async_wait.c b/crypto/async/async_wait.c
new file mode 100644
index 0000000000..9d90e524c3
--- /dev/null
+++ b/crypto/async/async_wait.c
@@ -0,0 +1,233 @@
+/*
+ * Written by Matt Caswell (matt@openssl.org) for the OpenSSL project.
+ */
+/* ====================================================================
+ * Copyright (c) 2016 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    licensing@OpenSSL.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ */
+
+/* This must be the first #include file */
+#include "async_locl.h"
+
+#include <openssl/err.h>
+
+ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void)
+{
+    return OPENSSL_zalloc(sizeof(ASYNC_WAIT_CTX));
+}
+
+void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr;
+
+    if (ctx == NULL)
+        return;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has already been deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->cleanup != NULL)
+            curr->cleanup(ctx, curr->key, curr->fd, curr->custom_data);
+        curr = curr->next;
+    }
+
+    OPENSSL_free(ctx);
+}
+int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                               OSSL_ASYNC_FD fd, void *custom_data,
+                               void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *))
+{
+    struct fd_lookup_st *fdlookup;
+
+    fdlookup = OPENSSL_zalloc(sizeof *fdlookup);
+    if (fdlookup == NULL)
+        return 0;
+
+    fdlookup->key = key;
+    fdlookup->fd = fd;
+    fdlookup->custom_data = custom_data;
+    fdlookup->cleanup = cleanup;
+    fdlookup->add = 1;
+    fdlookup->next = ctx->fds;
+    ctx->fds = fdlookup;
+    ctx->numadd++;
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                          OSSL_ASYNC_FD *fd, void **custom_data)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            *fd = curr->fd;
+            *custom_data = curr->custom_data;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                               size_t *numfds)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    *numfds = 0;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (fd != NULL) {
+            *fd = curr->fd;
+            fd++;
+        }
+        (*numfds)++;
+        curr = curr->next;
+    }
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                   size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                   size_t *numdelfds)
+{
+    struct fd_lookup_st *curr;
+
+    *numaddfds = ctx->numadd;
+    *numdelfds = ctx->numdel;
+    if (addfd == NULL && delfd == NULL)
+        return 1;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        /* We ignore fds that have been marked as both added and deleted */
+        if (curr->del && !curr->add && (delfd != NULL)) {
+            *delfd = curr->fd;
+            delfd++;
+        }
+        if (curr->add && !curr->del && (addfd != NULL)) {
+            *addfd = curr->fd;
+            addfd++;
+        }
+        curr = curr->next;
+    }
+
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted already so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            /*
+             * Mark it as deleted. We don't call cleanup if explicitly asked
+             * to clear an fd. We assume the caller is going to do that
+             */
+            curr->del = 1;
+            ctx->numdel++;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr, *prev = NULL;
+
+    ctx->numadd = 0;
+    ctx->numdel = 0;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        if (curr->del) {
+            if (prev == NULL)
+                ctx->fds = curr->next;
+            else
+                prev->next = curr->next;
+            OPENSSL_free(curr);
+            if (prev == NULL)
+                curr = ctx->fds;
+            else
+                curr = prev->next;
+            continue;
+        }
+        if (curr->add) {
+            curr->add = 0;
+        }
+        prev = curr;
+        curr = curr->next;
+    }
+}
diff --git a/crypto/async/build.info b/crypto/async/build.info
index d975003888..278e3e9f89 100644
--- a/crypto/async/build.info
+++ b/crypto/async/build.info
@@ -1,3 +1,4 @@
 LIBS=../../libcrypto
 SOURCE[../../libcrypto]=\
-        async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+        async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c \
+        arch/async_null.c
diff --git a/doc/crypto/ASYNC_WAIT_CTX_new.pod b/doc/crypto/ASYNC_WAIT_CTX_new.pod
new file mode 100644
index 0000000000..b70f730f5e
--- /dev/null
+++ b/doc/crypto/ASYNC_WAIT_CTX_new.pod
@@ -0,0 +1,125 @@
+=pod
+
+=head1 NAME
+
+ASYNC_WAIT_CTX_new, ASYNC_WAIT_CTX_free, ASYNC_WAIT_CTX_set_wait_fd,
+ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds, ASYNC_WAIT_CTX_clear_fd - functions to manage
+waiting for asynchronous jobs to complete
+
+=head1 SYNOPSIS
+
+ #include <openssl/async.h>
+
+ ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void);
+ void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx);
+ int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                                OSSL_ASYNC_FD fd,
+                                void *custom_data,
+                                void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *));
+ int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                           OSSL_ASYNC_FD *fd, void **custom_data);
+ int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                                size_t *numfds);
+ int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                    size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                    size_t *numdelfds);
+ int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key);
+
+
+=head1 DESCRIPTION
+
+For an overview of how asynchronous operations are implemented in OpenSSL see
+L<ASYNC_start_job(3)>. An ASYNC_WAIT_CTX object represents an asynchronous
+"session", i.e. a related set of crypto operations. For example in SSL terms
+this would have a one-to-one correspondence with an SSL connection.
+
+Application code must create an ASYNC_WAIT_CTX using the ASYNC_WAIT_CTX_new()
+function prior to calling ASYNC_start_job() (see L<ASYNC_start_job(3)>). When
+the job is started it is associated with the ASYNC_WAIT_CTX for the duration of
+that job. An ASYNC_WAIT_CTX should only be used for one ASYNC_JOB at any one
+time, but can be reused after an ASYNC_JOB has finished for a subsequent
+ASYNC_JOB. When the session is complete (e.g. the SSL connection is closed),
+application code cleans up with ASYNC_WAIT_CTX_free().
+
+ASYNC_WAIT_CTXs can have "wait" file descriptors associated with them. Calling
+ASYNC_WAIT_CTX_get_all_fds() and passing in a pointer to an ASYNC_WAIT_CTX in
+the B<ctx> parameter will return the wait file descriptors associated with that
+job in B<*fd>. The number of file descriptors returned will be stored in
+B<*numfds>. It is the caller's responsibility to ensure that sufficient memory
+has been allocated in B<*fd> to receive all the file descriptors. Calling
+ASYNC_WAIT_CTX_get_all_fds() with a NULL B<fd> value will return no file
+descriptors but will still populate B<*numfds>. Therefore application code is
+typically expected to call this function twice: once to get the number of fds,
+and then again when sufficient memory has been allocated. If only one
+asynchronous engine is being used then noramlly this call will only ever return
+one fd. If multiple asynchronous engines are being used then more could be
+returned.
+
+The function ASYNC_WAIT_CTX_fds_have_changed() can be used to detect if any fds
+have changed since the last call time ASYNC_start_job() returned an ASYNC_PAUSE
+result (or since the ASYNC_WAIT_CTX was created if no ASYNC_PAUSE result has
+been received). The B<numaddfds> and B<numdelfds> parameters will be populated
+with the number of fds added or deleted respectively. B<*addfd> and B<*delfd>
+will be populated with the list of added and deleted fds respectively. Similarly
+to ASYNC_WAIT_CTX_get_all_fds() either of these can be NULL, but if they are not
+NULL then the caller is responsible for ensuring sufficient memory is allocated.
+
+Implementors of async aware code (e.g. engines) are encouraged to return a
+stable fd for the lifetime of the ASYNC_WAIT_CTX in order to reduce the "churn"
+of regularly changing fds - although no guarantees of this are provided to
+applications.
+
+Applications can wait for the file descriptor to be ready for "read" using a
+system function call such as select or poll (being ready for "read" indicates
+that the job should be resumed). If no file descriptor is made available then an
+application will have to periodically "poll" the job by attempting to restart it
+to see if it is ready to continue.
+
+Async aware code (e.g. engines) can get the current ASYNC_WAIT_CTX from the job
+via L<ASYNC_get_async_wait_ctx(3)> and provide a file descriptor to use for
+waiting on by calling ASYNC_WAIT_CTX_set_wait_fd(). Typically this would be done
+by an engine immediately prior to calling ASYNC_pause_job() and not by end user
+code. An existing association with a file descriptor can be obtained using
+ASYNC_WAIT_CTX_get_fd() and cleared using ASYNC_WAIT_CTX_clear_fd(). Both of
+these functions requires a B<key> value which is unique to the async aware code.
+This could be any unique value but a good candidate might be the B<ENGINE *> for
+the engine. The B<custom_data> parameter can be any value, and will be returned
+in a subsequent call to ASYNC_WAIT_CTX_get_fd(). The
+ASYNC_WAIT_CTX_set_wait_fd() function also expects a pointer to a "cleanup"
+routine. This can be NULL but if provided will automatically get called when the
+ASYNC_WAIT_CTX is freed, and gives the engine the opportunity to close the fd or
+any other resources.
+
+An example of typical usage might be an async capable engine. User code would
+initiate cryptographic operations. The engine would initiate those operations
+asynchronously and then call ASYNC_WAIT_CTX_set_wait_fd() followed by
+ASYNC_pause_job() to return control to the user code. The user code can then
+perform other tasks or wait for the job to be ready by calling "select" or other
+similar function on the wait file descriptor. The engine can signal to the user
+code that the job should be resumed by making the wait file descriptor
+"readable". Once resumed the engine should clear the wake signal on the wait
+file descriptor.
+
+=head1 RETURN VALUES
+
+ASYNC_WAIT_CTX_new() returns a pointer to the newly allocated ASYNC_WAIT_CTX or
+NULL on error.
+
+ASYNC_WAIT_CTX_set_wait_fd, ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds and ASYNC_WAIT_CTX_clear_fd all return 1 on
+success or 0 on error.
+
+=head1 SEE ALSO
+
+L<crypto(3)>, L<ASYNC_start_job(3)>
+
+=head1 HISTORY
+
+ASYNC_WAIT_CTX_new, ASYNC_WAIT_CTX_free, ASYNC_WAIT_CTX_set_wait_fd,
+ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds, ASYNC_WAIT_CTX_clear_fd were first added to
+OpenSSL 1.1.0.
+
+=cut
diff --git a/doc/crypto/ASYNC_start_job.pod b/doc/crypto/ASYNC_start_job.pod
index 5297f86dfe..81fdc97df4 100644
--- a/doc/crypto/ASYNC_start_job.pod
+++ b/doc/crypto/ASYNC_start_job.pod
@@ -2,29 +2,24 @@
 
 =head1 NAME
 
-ASYNC_init, ASYNC_cleanup, ASYNC_init_thread, ASYNC_cleanup_thread,
-ASYNC_start_job, ASYNC_pause_job, ASYNC_in_job, ASYNC_get_wait_fd,
-ASYNC_get_current_job, ASYNC_wake, ASYNC_clear_wake, ASYNC_block_pause,
-ASYNC_unblock_pause - asynchronous job management functions
+ASYNC_init_thread, ASYNC_cleanup_thread, ASYNC_start_job, ASYNC_pause_job,
+ASYNC_in_job, ASYNC_get_wait_fd, ASYNC_set_wait_fd, ASYNC_clear_wait_fd,
+ASYNC_get_current_job, ASYNC_block_pause, ASYNC_unblock_pause - asynchronous job
+management functions
 
 =head1 SYNOPSIS
 
  #include <openssl/async.h>
 
- int ASYNC_init(int init_thread, size_t max_size, size_t init_size);
- void ASYNC_cleanup(int cleanupthread);
-
  int ASYNC_init_thread(size_t max_size, size_t init_size);
  void ASYNC_cleanup_thread(void);
 
- int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                     void *args, size_t size);
+ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *ctx, int *ret,
+                     int (*func)(void *), void *args, size_t size);
  int ASYNC_pause_job(void);
 
- int ASYNC_get_wait_fd(ASYNC_JOB *job);
  ASYNC_JOB *ASYNC_get_current_job(void);
- void ASYNC_wake(ASYNC_JOB *job);
- void ASYNC_clear_wake(ASYNC_JOB *job);
+ ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job);
  void ASYNC_block_pause(void);
  void ASYNC_unblock_pause(void);
 
@@ -38,19 +33,14 @@ subsequent event indicates that the job can be resumed.
 The creation of an ASYNC_JOB is a relatively expensive operation. Therefore, for
 efficiency reasons, jobs can be created up front and reused many times. They are
 held in a pool until they are needed, at which point they are removed from the
-pool, used, and then returned to the pool when the job completes. Before using
-any of the asynchronous job functions, user code should first call
-ASYNC_init(). If the user application is multi-threaded, then
-ASYNC_init_thread() should be called for each thread that will initiate
-asynchronous jobs. If the B<init_thread> parameter to ASYNC_init() is non-zero
-then ASYNC_init_thread is automatically called for the current thread. Before
-user code exits it should free up resources for each thread that was initialised
-using ASYNC_cleanup_thread(). No asynchronous jobs must be outstanding for the thread
-when ASYNC_cleanup_thread() is called. Failing to ensure this will result in memory
-leaks. Additionally an application should call ASYNC_cleanup() when all
-asynchronous work is complete across all threads. If B<cleanupthread> is
-non-zero then ASYNC_cleanup_thread() is automatically called for the current
-thread.
+pool, used, and then returned to the pool when the job completes. If the user
+application is multi-threaded, then ASYNC_init_thread() may be called for each
+thread that will initiate asynchronous jobs. Before
+user code exits per-thread resources need to be cleaned up. This will normally
+occur automatically (see L<OPENSSL_init_crypto(3)>) but may be explicitly
+initiated by using ASYNC_cleanup_thread(). No asynchronous jobs must be
+outstanding for the thread when ASYNC_cleanup_thread() is called. Failing to
+ensure this will result in memory leaks.
 
 The B<max_size> argument limits the number of ASYNC_JOBs that will be held in
 the pool. If B<max_size> is set to 0 then no upper limit is set. When an
@@ -60,16 +50,16 @@ pool does not exceed B<max_size>. When the pool is first initialised
 B<init_size> ASYNC_JOBs will be created immediately. If ASYNC_init_thread() is
 not called before the pool is first used then it will be called automatically
 with a B<max_size> of 0 (no upper limit) and an B<init_size> of 0 (no ASYNC_JOBs
-created up front). If a pool is created in this way it must still be cleaned up
-with an explicit call to ASYNC_cleanup_thread().
+created up front).
 
 An asynchronous job is started by calling the ASYNC_start_job() function.
-Initially B<*job> should be NULL. B<ret> should point to a location where the
-return value of the asynchronous function should be stored on completion of the
-job. B<func> represents the function that should be started asynchronously. The
-data pointed to by B<args> and of size B<size> will be copied and then passed as
-an argument to B<func> when the job starts. ASYNC_start_job will return one of
-the following values:
+Initially B<*job> should be NULL. B<ctx> should point to an ASYNC_WAIT_CTX
+object created through the L<ASYNC_WAIT_CTX_new(3)> function. B<ret> should
+point to a location where the return value of the asynchronous function should
+be stored on completion of the job. B<func> represents the function that should
+be started asynchronously. The data pointed to by B<args> and of size B<size>
+will be copied and then passed as an argument to B<func> when the job starts.
+ASYNC_start_job will return one of the following values:
 
 =over 4
 
@@ -114,23 +104,23 @@ B<*job> parameter will resume execution from the ASYNC_pause_job() call. If
 ASYNC_pause_job() is called whilst not within the context of a job then no
 action is taken and ASYNC_pause_job() returns immediately.
 
-Every ASYNC_JOB has a "wait" file descriptor associated with it. Calling
-ASYNC_get_wait_fd() and passing in a pointer to an ASYNC_JOB in the B<job>
-parameter will return the wait file descriptor associated with that job. This
-file descriptor can be used to signal that the job should be resumed.
-Applications can wait for the file descriptor to be ready for "read" using a
-system function call such as select or poll (being ready for "read" indicates
-that the job should be resumed). Applications can signal that a job is ready to
-resume using ASYNC_wake() or clear an existing signal using ASYNC_clear_wake().
+ASYNC_get_wait_ctx() can be used to get a pointer to the ASYNC_WAIT_CTX
+for the B<job>. ASYNC_WAIT_CTXs can have a "wait" file descriptor associated
+with them. Applications can wait for the file descriptor to be ready for "read"
+using a system function call such as select or poll (being ready for "read"
+indicates that the job should be resumed). If no file descriptor is made
+available then an application will have to priodically "poll" the job by
+attempting to restart it to see if it is ready to continue.
 
 An example of typical usage might be an async capable engine. User code would
 initiate cryptographic operations. The engine would initiate those operations
-asynchronously and then call ASYNC_pause_job() to return control to the user
-code. The user code can then perform other