asymciphers: add FIPS error state handling

Check for provider being runnable in newctx, init, encrypt and decrypt.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12801)

1 files changed, 18 insertions, 2 deletions

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index 817f705b53..f53284ca49 100644
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -28,6 +28,7 @@
 #include "prov/providercommonerr.h"
 #include "prov/provider_ctx.h"
 #include "prov/implementations.h"
+#include "prov/providercommon.h"
 
 #include <stdlib.h>
 
@@ -77,8 +78,11 @@ typedef struct {
 
 static void *rsa_newctx(void *provctx)
 {
-    PROV_RSA_CTX *prsactx =  OPENSSL_zalloc(sizeof(PROV_RSA_CTX));
+    PROV_RSA_CTX *prsactx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+    prsactx = OPENSSL_zalloc(sizeof(PROV_RSA_CTX));
     if (prsactx == NULL)
         return NULL;
     prsactx->libctx = PROV_LIBRARY_CONTEXT_OF(provctx);
@@ -90,7 +94,10 @@ static int rsa_init(void *vprsactx, void *vrsa)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 
-    if (prsactx == NULL || vrsa == NULL || !RSA_up_ref(vrsa))
+    if (!ossl_prov_is_running()
+            || prsactx == NULL
+            || vrsa == NULL
+            || !RSA_up_ref(vrsa))
         return 0;
     RSA_free(prsactx->rsa);
     prsactx->rsa = vrsa;
@@ -113,6 +120,9 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
     int ret;
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (out == NULL) {
         size_t len = RSA_size(prsactx->rsa);
 
@@ -171,6 +181,9 @@ static int rsa_decrypt(void *vprsactx, unsigned char *out, size_t *outlen,
     int ret;
     size_t len = RSA_size(prsactx->rsa);
 
+    if (!ossl_prov_is_running())
+        return 0;
+
     if (prsactx->pad_mode == RSA_PKCS1_WITH_TLS_PADDING) {
         if (out == NULL) {
             *outlen = SSL_MAX_MASTER_KEY_LENGTH;
@@ -269,6 +282,9 @@ static void *rsa_dupctx(void *vprsactx)
     PROV_RSA_CTX *srcctx = (PROV_RSA_CTX *)vprsactx;
     PROV_RSA_CTX *dstctx;
 
+    if (!ossl_prov_is_running())
+        return NULL;
+
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
     if (dstctx == NULL)
         return NULL;