Implement windows async thread local variable support

Implements Thread Local Storage in the windows async port. This also has
some knock on effects to the posix and null implementations.

Reviewed-by: Rich Salz <rsalz@openssl.org>

4 files changed, 113 insertions, 14 deletions

diff --git a/crypto/async/arch/async_null.c b/crypto/async/arch/async_null.c
index dba159f309..b2dbfee7ec 100644
--- a/crypto/async/arch/async_null.c
+++ b/crypto/async/arch/async_null.c
@@ -76,10 +76,23 @@ int async_read1(OSSL_ASYNC_FD fd, void *buf)
     return -1;
 }
 
-int async_thread_local_init(void)
+int async_global_init(void)
 {
     return 0;
 }
 
+int async_local_init(void)
+{
+    return 0;
+}
+
+void async_local_cleanup(void)
+{
+}
+
+void async_global_cleanup(void)
+{
+}
+
 #endif
 
diff --git a/crypto/async/arch/async_posix.c b/crypto/async/arch/async_posix.c
index bd4b0c2f1b..77a2c33de6 100644
--- a/crypto/async/arch/async_posix.c
+++ b/crypto/async/arch/async_posix.c
@@ -66,7 +66,7 @@ pthread_key_t posixpool;
 
 #define STACKSIZE       32768
 
-int async_thread_local_init(void)
+int async_global_init(void)
 {
     if (pthread_key_create(&posixctx, NULL) != 0
             || pthread_key_create(&posixpool, NULL) != 0)
@@ -75,6 +75,22 @@ int async_thread_local_init(void)
     return 1;
 }
 
+int async_local_init(void)
+{
+    if (!async_set_ctx(NULL) || ! async_set_pool(NULL))
+        return 0;
+
+    return 1;
+}
+
+void async_local_cleanup(void)
+{
+}
+
+void async_global_cleanup(void)
+{
+}
+
 int async_fibre_init(async_fibre *fibre)
 {
     void *stack = NULL;
diff --git a/crypto/async/arch/async_win.c b/crypto/async/arch/async_win.c
index 4eb449d34f..20c8a09bc4 100644
--- a/crypto/async/arch/async_win.c
+++ b/crypto/async/arch/async_win.c
@@ -64,18 +64,80 @@ struct winpool {
     size_t max_size;
 };
 
+static DWORD asyncwinpool = 0;
+static DWORD asyncwinctx = 0;
+static DWORD asyncwindispatch = 0;
+
+
 void async_start_func(void);
 
+int async_global_init(void)
+{
+    asyncwinpool = TlsAlloc();
+    asyncwinctx = TlsAlloc();
+    asyncwindispatch = TlsAlloc();
+    if (asyncwinpool == TLS_OUT_OF_INDEXES || asyncwinctx == TLS_OUT_OF_INDEXES
+            || asyncwindispatch == TLS_OUT_OF_INDEXES) {
+        if (asyncwinpool != TLS_OUT_OF_INDEXES) {
+            TlsFree(asyncwinpool);
+        }
+        if (asyncwinctx != TLS_OUT_OF_INDEXES) {
+            TlsFree(asyncwinctx);
+        }
+        if (asyncwindispatch != TLS_OUT_OF_INDEXES) {
+            TlsFree(asyncwindispatch);
+        }
+        return 0;
+    }
+    return 1;
+}
+
+int async_local_init(void)
+{
+    return (TlsSetValue(asyncwinpool, NULL) != 0)
+        && (TlsSetValue(asyncwinctx, NULL) != 0)
+        && (TlsSetValue(asyncwindispatch, NULL) != 0);
+}
+
+void async_local_cleanup(void)
+{
+    async_ctx *ctx = async_get_ctx();
+    if (ctx != NULL) {
+        async_fibre *fibre = &ctx->dispatcher;
+        if(fibre != NULL && fibre->fibre != NULL && fibre->converted) {
+            ConvertFiberToThread();
+            fibre->fibre = NULL;
+        }
+    }
+}
+
+void async_global_cleanup(void)
+{
+    TlsFree(asyncwinpool);
+    TlsFree(asyncwinctx);
+    TlsFree(asyncwindispatch);
+    asyncwinpool = 0;
+    asyncwinctx = 0;
+    asyncwindispatch = 0;
+}
+
 int async_fibre_init_dispatcher(async_fibre *fibre)
 {
     LPVOID dispatcher;
 
-    dispatcher =
-        (LPVOID) CRYPTO_get_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_DISPATCH);
+    dispatcher = (LPVOID)TlsGetValue(asyncwindispatch);
     if (dispatcher == NULL) {
         fibre->fibre = ConvertThreadToFiber(NULL);
-        CRYPTO_set_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_DISPATCH,
-                                (void *)fibre->fibre);
+        if (fibre->fibre == NULL) {
+            fibre->converted = 0;
+            fibre->fibre = GetCurrentFiber();
+            if (fibre->fibre == NULL)
+                return 0;
+        } else {
+            fibre->converted = 1;
+        }
+        if (TlsSetValue(asyncwindispatch, (LPVOID)fibre->fibre) == 0)
+            return 0;
     } else {
         fibre->fibre = dispatcher;
     }
@@ -125,15 +187,23 @@ int async_read1(OSSL_ASYNC_FD fd, void *buf)
 
 async_pool *async_get_pool(void)
 {
-    return (async_pool *)
-            CRYPTO_get_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_POOL);
+    return (async_pool *)TlsGetValue(asyncwinpool);
 }
 
 
 int async_set_pool(async_pool *pool)
 {
-    CRYPTO_set_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_POOL, (void *)pool);
-    return 1;
+    return TlsSetValue(asyncwinpool, (LPVOID)pool) != 0;
+}
+
+async_ctx *async_get_ctx(void)
+{
+    return (async_ctx *)TlsGetValue(asyncwinctx);
+}
+
+int async_set_ctx(async_ctx *ctx)
+{
+    return TlsSetValue(asyncwinctx, (LPVOID)ctx) != 0;
 }
 
 #endif
diff --git a/crypto/async/arch/async_win.h b/crypto/async/arch/async_win.h
index 5e91732e87..77e41e405b 100644
--- a/crypto/async/arch/async_win.h
+++ b/crypto/async/arch/async_win.h
@@ -66,18 +66,18 @@
 
 typedef struct async_fibre_st {
     LPVOID fibre;
+    int converted;
 } async_fibre;
 
-# define async_set_ctx(nctx) \
-        (CRYPTO_set_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_CTX, (void *)(nctx)))
-# define async_get_ctx() \
-        ((async_ctx *)CRYPTO_get_thread_local(CRYPTO_THREAD_LOCAL_ASYNC_CTX))
 # define async_fibre_swapcontext(o,n,r) \
         (SwitchToFiber((n)->fibre), 1)
 # define async_fibre_makecontext(c) \
         ((c)->fibre = CreateFiber(0, async_start_func_win, 0))
 # define async_fibre_free(f)             (DeleteFiber((f)->fibre))
 
+async_ctx *async_get_ctx(void);
+int async_set_ctx(async_ctx *ctx);
+
 int async_fibre_init_dispatcher(async_fibre *fibre);
 VOID CALLBACK async_start_func_win(PVOID unused);