Refactor the async wait fd logic

Implementation experience has shown that the original plan for async wait
fds was too simplistic. Originally the async logic created a pipe internally
and user/engine code could then get access to it via API calls. It is more
flexible if the engine is able to create its own fd and provide it to the
async code.

Another issue is that there can be a lot of churn in the fd value within
the context of (say) a single SSL connection leading to continually adding
and removing fds from (say) epoll. It is better if we can provide some
stability of the fd value across a whole SSL connection. This is
problematic because an engine has no concept of an SSL connection.

This commit refactors things to introduce an ASYNC_WAIT_CTX which acts as a
proxy for an SSL connection down at the engine layer.

Reviewed-by: Richard Levitte <levitte@openssl.org>

1 files changed, 87 insertions, 52 deletions

diff --git a/test/asynctest.c b/test/asynctest.c
index ddff70cde6..bfdf441e7c 100644
--- a/test/asynctest.c
+++ b/test/asynctest.c
@@ -103,12 +103,24 @@ static int save_current(void *args)
     return 1;
 }
 
-static int wake(void *args)
+#define MAGIC_WAIT_FD   ((OSSL_ASYNC_FD)99)
+static int waitfd(void *args)
 {
+    ASYNC_JOB *job;
+    ASYNC_WAIT_CTX *waitctx;
     ASYNC_pause_job();
-    ASYNC_wake(ASYNC_get_current_job());
+    job = ASYNC_get_current_job();
+    if (job == NULL)
+        return 0;
+    waitctx = ASYNC_get_wait_ctx(job);
+    if (waitctx == NULL)
+        return 0;
+    if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, waitctx, MAGIC_WAIT_FD, NULL, NULL))
+        return 0;
     ASYNC_pause_job();
-    ASYNC_clear_wake(ASYNC_get_current_job());
+
+    if (!ASYNC_WAIT_CTX_clear_fd(waitctx, waitctx))
+        return 0;
 
     return 1;
 }
@@ -127,30 +139,34 @@ static int test_ASYNC_init_thread()
 {
     ASYNC_JOB *job1 = NULL, *job2 = NULL, *job3 = NULL;
     int funcret1, funcret2, funcret3;
+    ASYNC_WAIT_CTX *waitctx;
 
     if (       !ASYNC_init_thread(2, 0)
-            || ASYNC_start_job(&job1, &funcret1, only_pause, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job1, waitctx, &funcret1, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job2, &funcret2, only_pause, NULL, 0)
+            || ASYNC_start_job(&job2, waitctx, &funcret2, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_NO_JOBS
-            || ASYNC_start_job(&job1, &funcret1, only_pause, NULL, 0)
+            || ASYNC_start_job(&job1, waitctx, &funcret1, only_pause, NULL, 0)
                 != ASYNC_FINISH
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job2, &funcret2, only_pause, NULL, 0)
+            || ASYNC_start_job(&job2, waitctx, &funcret2, only_pause, NULL, 0)
                 != ASYNC_FINISH
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_FINISH
             || funcret1 != 1
             || funcret2 != 1
             || funcret3 != 1) {
         fprintf(stderr, "test_ASYNC_init_thread() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -159,20 +175,26 @@ static int test_ASYNC_start_job()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     ctr = 0;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_PAUSE
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, add_two, NULL, 0)
+               != ASYNC_PAUSE
             || ctr != 1
-            || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_FINISH
+            || ASYNC_start_job(&job, waitctx, &funcret, add_two, NULL, 0)
+               != ASYNC_FINISH
             || ctr != 2
             || funcret != 2) {
         fprintf(stderr, "test_ASYNC_start_job() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -181,74 +203,83 @@ static int test_ASYNC_get_current_job()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     currjob = NULL;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, save_current, NULL, 0)
                 != ASYNC_PAUSE
             || currjob != job
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || ASYNC_start_job(&job, waitctx, &funcret, save_current, NULL, 0)
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_current_job() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
 
-static int hasdata(OSSL_ASYNC_FD fd)
-{
-#ifdef ASYNC_POSIX
-    fd_set checkfds;
-    struct timeval tv;
-    FD_ZERO(&checkfds);
-    openssl_fdset(fd, &checkfds);
-    memset(&tv, 0, sizeof tv);
-    if (select(fd + 1, (void *)&checkfds, NULL, NULL, &tv) < 0)
-        return -1;
-    if (FD_ISSET(fd, &checkfds))
-        return 1;
-    return 0;
-#else
-    DWORD avail = 0;
-
-    if (PeekNamedPipe(fd, NULL, 0, NULL, &avail, NULL) && avail > 0)
-        return 1;
-
-    return 0;
-#endif
-}
-
-static int test_ASYNC_get_wait_fd()
+static int test_ASYNC_WAIT_CTX_get_all_fds()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
-    OSSL_ASYNC_FD fd;
+    ASYNC_WAIT_CTX *waitctx;
+    OSSL_ASYNC_FD fd = OSSL_BAD_ASYNC_FD, delfd = OSSL_BAD_ASYNC_FD;
+    size_t numfds, numdelfds;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, wake, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+               /* On first run we're not expecting any wait fds */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_PAUSE
-            || (fd = ASYNC_get_wait_fd(job)) < 0
-            || hasdata(fd) != 0
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                               &numdelfds)
+            || numfds != 0
+            || numdelfds != 0
+               /* On second run we're expecting one added fd */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_PAUSE
-            || hasdata(fd) != 1
-            || (ASYNC_clear_wake(job), 0)
-            || hasdata(fd) != 0
-            || (ASYNC_wake(job), 0)
-            || hasdata(fd) != 1
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 1
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, &fd, &numfds)
+            || fd != MAGIC_WAIT_FD
+            || (fd = OSSL_BAD_ASYNC_FD, 0) /* Assign to something else */
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                              &numdelfds)
+            || numfds != 1
+            || numdelfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, &fd, &numfds, NULL,
+                                               &numdelfds)
+            || fd != MAGIC_WAIT_FD
+               /* On final run we expect one deleted fd */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_FINISH
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                               &numdelfds)
+            || numfds != 0
+            || numdelfds != 1
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, &delfd,
+                                               &numdelfds)
+            || delfd != MAGIC_WAIT_FD
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_wait_fd() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -257,18 +288,22 @@ static int test_ASYNC_block_pause()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, blockpause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
+            || ASYNC_start_job(&job, waitctx, &funcret, blockpause, NULL, 0)
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_block_pause() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -287,7 +322,7 @@ int main(int argc, char **argv)
     if (       !test_ASYNC_init_thread()
             || !test_ASYNC_start_job()
             || !test_ASYNC_get_current_job()
-            || !test_ASYNC_get_wait_fd()
+            || !test_ASYNC_WAIT_CTX_get_all_fds()
             || !test_ASYNC_block_pause()) {
         return 1;
     }