Convert thread stop handling into a publish/subscribe model

In later commits this will allow providers to subscribe to thread stop
events. We will need this in the FIPS module. We also make thread stop
handling OPENSSL_CTX aware (different OPENSSL_CTXs may have different
thread local data that needs cleaning up).

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9040)

8 files changed, 64 insertions, 75 deletions

diff --git a/crypto/async/async.c b/crypto/async/async.c
index 53d288ca62..785ed06906 100644
--- a/crypto/async/async.c
+++ b/crypto/async/async.c
@@ -30,11 +30,13 @@
 static CRYPTO_THREAD_LOCAL ctxkey;
 static CRYPTO_THREAD_LOCAL poolkey;
 
+static void async_delete_thread_state(OPENSSL_CTX *ctx);
+
 static async_ctx *async_ctx_new(void)
 {
     async_ctx *nctx;
 
-    if (!ossl_init_thread_start(OPENSSL_INIT_THREAD_ASYNC))
+    if (!ossl_init_thread_start(NULL, async_delete_thread_state))
         return NULL;
 
     nctx = OPENSSL_malloc(sizeof(*nctx));
@@ -326,7 +328,7 @@ int ASYNC_init_thread(size_t max_size, size_t init_size)
     if (!OPENSSL_init_crypto(OPENSSL_INIT_ASYNC, NULL))
         return 0;
 
-    if (!ossl_init_thread_start(OPENSSL_INIT_THREAD_ASYNC))
+    if (!ossl_init_thread_start(NULL, async_delete_thread_state))
         return 0;
 
     pool = OPENSSL_zalloc(sizeof(*pool));
@@ -374,7 +376,8 @@ err:
     return 0;
 }
 
-void async_delete_thread_state(void)
+/* OPENSSL_CTX ignored for now */
+static void async_delete_thread_state(OPENSSL_CTX *ctx)
 {
     async_pool *pool = (async_pool *)CRYPTO_THREAD_get_local(&poolkey);
 
@@ -393,7 +396,7 @@ void ASYNC_cleanup_thread(void)
     if (!OPENSSL_init_crypto(OPENSSL_INIT_ASYNC, NULL))
         return;
 
-    async_delete_thread_state();
+    async_delete_thread_state(NULL);
 }
 
 ASYNC_JOB *ASYNC_get_current_job(void)
diff --git a/crypto/err/err.c b/crypto/err/err.c
index cf3ae4d3b3..2559e76baf 100644
--- a/crypto/err/err.c
+++ b/crypto/err/err.c
@@ -688,7 +688,8 @@ const char *ERR_reason_error_string(unsigned long e)
     return ((p == NULL) ? NULL : p->string);
 }
 
-void err_delete_thread_state(void)
+/* OPENSSL_CTX ignored for now */
+static void err_delete_thread_state(OPENSSL_CTX *ctx)
 {
     ERR_STATE *state = CRYPTO_THREAD_get_local(&err_thread_local);
     if (state == NULL)
@@ -740,7 +741,7 @@ ERR_STATE *ERR_get_state(void)
             return NULL;
         }
 
-        if (!ossl_init_thread_start(OPENSSL_INIT_THREAD_ERR_STATE)
+        if (!ossl_init_thread_start(NULL, err_delete_thread_state)
                 || !CRYPTO_THREAD_set_local(&err_thread_local, state)) {
             ERR_STATE_free(state);
             CRYPTO_THREAD_set_local(&err_thread_local, NULL);
diff --git a/crypto/include/internal/async.h b/crypto/include/internal/async.h
index f5454f2e87..e9a89da314 100644
--- a/crypto/include/internal/async.h
+++ b/crypto/include/internal/async.h
@@ -11,5 +11,4 @@
 
 int async_init(void);
 void async_deinit(void);
-void async_delete_thread_state(void);
 
diff --git a/crypto/include/internal/cryptlib_int.h b/crypto/include/internal/cryptlib_int.h
index 422ef01dde..b3529dcaf9 100644
--- a/crypto/include/internal/cryptlib_int.h
+++ b/crypto/include/internal/cryptlib_int.h
@@ -11,13 +11,9 @@
 
 /* This file is not scanned by mkdef.pl, whereas cryptlib.h is */
 
-struct thread_local_inits_st {
-    int async;
-    int err_state;
-    int rand;
-};
-
-int ossl_init_thread_start(uint64_t opts);
+typedef void (*ossl_thread_stop_handler_fn)(OPENSSL_CTX *ctx);
+int ossl_init_thread_start(OPENSSL_CTX *ctx,
+                           ossl_thread_stop_handler_fn handfn);
 
 /*
  * OPENSSL_INIT flags. The primary list of these is in crypto.h. Flags below
@@ -27,11 +23,6 @@ int ossl_init_thread_start(uint64_t opts);
 # define OPENSSL_INIT_ZLIB                   0x00010000L
 # define OPENSSL_INIT_BASE_ONLY              0x00040000L
 
-/* OPENSSL_INIT_THREAD flags */
-# define OPENSSL_INIT_THREAD_ASYNC           0x01
-# define OPENSSL_INIT_THREAD_ERR_STATE       0x02
-# define OPENSSL_INIT_THREAD_RAND            0x04
-
 int ossl_trace_init(void);
 void ossl_trace_cleanup(void);
 void ossl_malloc_setup_failures(void);
diff --git a/crypto/include/internal/err_int.h b/crypto/include/internal/err_int.h
index a2006cad7c..68c6d62e30 100644
--- a/crypto/include/internal/err_int.h
+++ b/crypto/include/internal/err_int.h
@@ -12,7 +12,6 @@
 
 int err_load_crypto_strings_int(void);
 void err_cleanup(void);
-void err_delete_thread_state(void);
 int err_shelve_state(void **);
 void err_unshelve_state(void *);
 
diff --git a/crypto/include/internal/rand_int.h b/crypto/include/internal/rand_int.h
index 53896cea76..c1e5e033bb 100644
--- a/crypto/include/internal/rand_int.h
+++ b/crypto/include/internal/rand_int.h
@@ -24,7 +24,6 @@
 typedef struct rand_pool_st RAND_POOL;
 
 void rand_cleanup_int(void);
-void drbg_delete_thread_state(void);
 void rand_fork(void);
 
 /* Hardware-based seeding functions. */
diff --git a/crypto/init.c b/crypto/init.c
index e73c9bafbd..7512a1e9d6 100644
--- a/crypto/init.c
+++ b/crypto/init.c
@@ -31,6 +31,13 @@
 
 static int stopped = 0;
 
+typedef struct thread_event_handler_st THREAD_EVENT_HANDLER;
+struct thread_event_handler_st {
+    OPENSSL_CTX *ctx;
+    ossl_thread_stop_handler_fn handfn;
+    THREAD_EVENT_HANDLER *next;
+};
+
 /*
  * Since per-thread-specific-data destructors are not universally
  * available, i.e. not on Windows, only below CRYPTO_THREAD_LOCAL key
@@ -50,30 +57,30 @@ static union {
     CRYPTO_THREAD_LOCAL value;
 } destructor_key = { -1 };
 
-static void ossl_init_thread_stop(struct thread_local_inits_st *locals);
+static void ossl_init_thread_stop(THREAD_EVENT_HANDLER **hands);
 
-static void ossl_init_thread_destructor(void *local)
+static void ossl_init_thread_destructor(void *hands)
 {
-    ossl_init_thread_stop((struct thread_local_inits_st *)local);
+    ossl_init_thread_stop((THREAD_EVENT_HANDLER **)hands);
 }
 
-static struct thread_local_inits_st *ossl_init_get_thread_local(int alloc)
+static THREAD_EVENT_HANDLER **ossl_init_get_thread_local(int alloc)
 {
-    struct thread_local_inits_st *local =
+    THREAD_EVENT_HANDLER **hands =
         CRYPTO_THREAD_get_local(&destructor_key.value);
 
     if (alloc) {
-        if (local == NULL
-            && (local = OPENSSL_zalloc(sizeof(*local))) != NULL
-            && !CRYPTO_THREAD_set_local(&destructor_key.value, local)) {
-            OPENSSL_free(local);
+        if (hands == NULL
+            && (hands = OPENSSL_zalloc(sizeof(*hands))) != NULL
+            && !CRYPTO_THREAD_set_local(&destructor_key.value, hands)) {
+            OPENSSL_free(hands);
             return NULL;
         }
     } else {
         CRYPTO_THREAD_set_local(&destructor_key.value, NULL);
     }
 
-    return local;
+    return hands;
 }
 
 typedef struct ossl_init_stop_st OPENSSL_INIT_STOP;
@@ -417,28 +424,23 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_zlib)
 }
 #endif
 
-static void ossl_init_thread_stop(struct thread_local_inits_st *locals)
+static void ossl_init_thread_stop(THREAD_EVENT_HANDLER **hands)
 {
+    THREAD_EVENT_HANDLER *curr, *prev = NULL;
+
     /* Can't do much about this */
-    if (locals == NULL)
+    if (hands == NULL)
         return;
 
-    if (locals->async) {
-        OSSL_TRACE(INIT, "async_delete_thread_state()\n");
-        async_delete_thread_state();
-    }
-
-    if (locals->err_state) {
-        OSSL_TRACE(INIT, "err_delete_thread_state()\n");
-        err_delete_thread_state();
-    }
-
-    if (locals->rand) {
-        OSSL_TRACE(INIT, "drbg_delete_thread_state()\n");
-        drbg_delete_thread_state();
+    curr = *hands;
+    while (curr != NULL) {
+        curr->handfn(curr->ctx);
+        prev = curr;
+        curr = curr->next;
+        OPENSSL_free(prev);
     }
 
-    OPENSSL_free(locals);
+    OPENSSL_free(hands);
 }
 
 void OPENSSL_thread_stop(void)
@@ -447,38 +449,27 @@ void OPENSSL_thread_stop(void)
         ossl_init_thread_stop(ossl_init_get_thread_local(0));
 }
 
-int ossl_init_thread_start(uint64_t opts)
+int ossl_init_thread_start(OPENSSL_CTX *ctx, ossl_thread_stop_handler_fn handfn)
 {
-    struct thread_local_inits_st *locals;
+    THREAD_EVENT_HANDLER **hands;
+    THREAD_EVENT_HANDLER *hand;
 
     if (!OPENSSL_init_crypto(0, NULL))
         return 0;
 
-    locals = ossl_init_get_thread_local(1);
+    hands = ossl_init_get_thread_local(1);
 
-    if (locals == NULL)
+    if (hands == NULL)
         return 0;
 
-    if (opts & OPENSSL_INIT_THREAD_ASYNC) {
-        OSSL_TRACE(INIT,
-                   "ossl_init_thread_start: "
-                   "marking thread for async\n");
-        locals->async = 1;
-    }
-
-    if (opts & OPENSSL_INIT_THREAD_ERR_STATE) {
-        OSSL_TRACE(INIT,
-                   "ossl_init_thread_start: "
-                   "marking thread for err_state\n");
-        locals->err_state = 1;
-    }
+    hand = OPENSSL_malloc(sizeof(*hand));
+    if (hand == NULL)
+        return 0;
 
-    if (opts & OPENSSL_INIT_THREAD_RAND) {
-        OSSL_TRACE(INIT,
-                   "ossl_init_thread_start: "
-                   "marking thread for rand\n");
-        locals->rand = 1;
-    }
+    hand->handfn = handfn;
+    hand->ctx = ctx;
+    hand->next = *hands;
+    *hands = hand;
 
     return 1;
 }
diff --git a/crypto/rand/drbg_lib.c b/crypto/rand/drbg_lib.c
index 26e2ccb152..79d986ef48 100644
--- a/crypto/rand/drbg_lib.c
+++ b/crypto/rand/drbg_lib.c
@@ -158,6 +158,14 @@ static void *drbg_ossl_ctx_new(OPENSSL_CTX *libctx)
     if (dgbl == NULL)
         return NULL;
 
+#ifndef FIPS_MODE
+    /*
+     * We need to ensure that base libcrypto thread handling has been
+     * initialised.
+     */
+     OPENSSL_init_crypto(0, NULL);
+#endif
+
     if (!CRYPTO_THREAD_init_local(&dgbl->private_drbg, NULL))
         goto err1;
 
@@ -1137,10 +1145,8 @@ err:
     return NULL;
 }
 
-void drbg_delete_thread_state(void)
+static void drbg_delete_thread_state(OPENSSL_CTX *ctx)
 {
-    /* TODO(3.0): Other PRs will pass the ctx as a param to this function */
-    OPENSSL_CTX *ctx = NULL;
     DRBG_GLOBAL *dgbl = drbg_get_global(ctx);
     RAND_DRBG *drbg;
 
@@ -1332,7 +1338,7 @@ RAND_DRBG *OPENSSL_CTX_get0_public_drbg(OPENSSL_CTX *ctx)
 
     drbg = CRYPTO_THREAD_get_local(&dgbl->public_drbg);
     if (drbg == NULL) {
-        if (!ossl_init_thread_start(OPENSSL_INIT_THREAD_RAND))
+        if (!ossl_init_thread_start(NULL, drbg_delete_thread_state))
             return NULL;
         drbg = drbg_setup(ctx, dgbl->master_drbg, RAND_DRBG_TYPE_PUBLIC);
         CRYPTO_THREAD_set_local(&dgbl->public_drbg, drbg);
@@ -1359,7 +1365,7 @@ RAND_DRBG *OPENSSL_CTX_get0_private_drbg(OPENSSL_CTX *ctx)
 
     drbg = CRYPTO_THREAD_get_local(&dgbl->private_drbg);
     if (drbg == NULL) {
-        if (!ossl_init_thread_start(OPENSSL_INIT_THREAD_RAND))
+        if (!ossl_init_thread_start(NULL, drbg_delete_thread_state))
             return NULL;
         drbg = drbg_setup(ctx, dgbl->master_drbg, RAND_DRBG_TYPE_PRIVATE);
         CRYPTO_THREAD_set_local(&dgbl->private_drbg, drbg);