Fix the incorrect checks of EVP_CIPHER_CTX_set_key_length

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18397)

(cherry picked from commit 8d9fec1781751d2106d899c6076eeb3da6930bfe)

6 files changed, 9 insertions, 7 deletions

diff --git a/apps/speed.c b/apps/speed.c
index dbeb371194..f16c78baab 100644
--- a/apps/speed.c
+++ b/apps/speed.c
@@ -691,7 +691,7 @@ static EVP_CIPHER_CTX *init_evp_cipher_ctx(const char *ciphername,
         goto end;
     }
 
-    if (!EVP_CIPHER_CTX_set_key_length(ctx, keylen)) {
+    if (EVP_CIPHER_CTX_set_key_length(ctx, keylen) <= 0) {
         EVP_CIPHER_CTX_free(ctx);
         ctx = NULL;
         goto end;
diff --git a/crypto/cmac/cmac.c b/crypto/cmac/cmac.c
index 218eb94259..15968f74c4 100644
--- a/crypto/cmac/cmac.c
+++ b/crypto/cmac/cmac.c
@@ -137,9 +137,9 @@ int CMAC_Init(CMAC_CTX *ctx, const void *key, size_t keylen,
 
         /* If anything fails then ensure we can't use this ctx */
         ctx->nlast_block = -1;
-        if (!EVP_CIPHER_CTX_get0_cipher(ctx->cctx))
+        if (EVP_CIPHER_CTX_get0_cipher(ctx->cctx) == NULL)
             return 0;
-        if (!EVP_CIPHER_CTX_set_key_length(ctx->cctx, keylen))
+        if (EVP_CIPHER_CTX_set_key_length(ctx->cctx, keylen) <= 0)
             return 0;
         if (!EVP_EncryptInit_ex(ctx->cctx, NULL, NULL, key, zero_iv))
             return 0;
diff --git a/crypto/evp/p_open.c b/crypto/evp/p_open.c
index b08f271642..92fd20f6aa 100644
--- a/crypto/evp/p_open.c
+++ b/crypto/evp/p_open.c
@@ -50,7 +50,7 @@ int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
     if (EVP_PKEY_decrypt(pctx, key, &keylen, ek, ekl) <= 0)
         goto err;
 
-    if (!EVP_CIPHER_CTX_set_key_length(ctx, keylen)
+    if (EVP_CIPHER_CTX_set_key_length(ctx, keylen) <= 0
         || !EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv))
         goto err;
 
diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index 441bf78bba..4a13070a0a 100644
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -612,7 +612,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
              * length. The key length is determined by the size of the
              * decrypted RSA key.
              */
-            if (!EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen)) {
+            if (EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen) <= 0) {
                 /* Use random key as MMA defence */
                 OPENSSL_clear_free(ek, eklen);
                 ek = tkey;
diff --git a/providers/implementations/kdfs/krb5kdf.c b/providers/implementations/kdfs/krb5kdf.c
index 2c887f0eb9..4000703ca9 100644
--- a/providers/implementations/kdfs/krb5kdf.c
+++ b/providers/implementations/kdfs/krb5kdf.c
@@ -336,8 +336,10 @@ static int cipher_init(EVP_CIPHER_CTX *ctx,
     klen = EVP_CIPHER_CTX_get_key_length(ctx);
     if (key_len != (size_t)klen) {
         ret = EVP_CIPHER_CTX_set_key_length(ctx, key_len);
-        if (!ret)
+        if (ret <= 0) {
+            ret = 0;
             goto out;
+        }
     }
     /* we never want padding, either the length requested is a multiple of
      * the cipher block size or we are passed a cipher that can cope with
diff --git a/test/aesgcmtest.c b/test/aesgcmtest.c
index c371f4754e..119d316a26 100644
--- a/test/aesgcmtest.c
+++ b/test/aesgcmtest.c
@@ -111,7 +111,7 @@ static int badkeylen_test(void)
     ret = TEST_ptr(cipher = EVP_aes_192_gcm())
           && TEST_ptr(ctx = EVP_CIPHER_CTX_new())
           && TEST_true(EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL))
-          && TEST_false(EVP_CIPHER_CTX_set_key_length(ctx, 2));
+          && TEST_int_le(EVP_CIPHER_CTX_set_key_length(ctx, 2), 0);
     EVP_CIPHER_CTX_free(ctx);
     return ret;
 }