Add notes about ignoring initialization failures on contexts

Fixes #20130

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/20136)

(cherry picked from commit d4c5d8ff483d99f94d649fb67f1f26fce9694c92)

4 files changed, 58 insertions, 8 deletions

diff --git a/doc/man3/EVP_DigestInit.pod b/doc/man3/EVP_DigestInit.pod
index ed1a5a34b8..27c07be788 100644
--- a/doc/man3/EVP_DigestInit.pod
+++ b/doc/man3/EVP_DigestInit.pod
@@ -566,6 +566,7 @@ Returns 1 for success or 0 for failure.
 EVP_Digest(),
 EVP_DigestInit_ex2(),
 EVP_DigestInit_ex(),
+EVP_DigestInit(),
 EVP_DigestUpdate(),
 EVP_DigestFinal_ex(),
 EVP_DigestFinalXOF(), and
@@ -640,6 +641,13 @@ are still in common use.
 For most applications the I<impl> parameter to EVP_DigestInit_ex() will be
 set to NULL to use the default digest implementation.
 
+Ignoring failure returns of EVP_DigestInit_ex(), EVP_DigestInit_ex2(), or
+EVP_DigestInit() can lead to undefined behavior on subsequent calls
+updating or finalizing the B<EVP_MD_CTX> such as the EVP_DigestUpdate() or
+EVP_DigestFinal() functions. The only valid calls on the B<EVP_MD_CTX>
+when initialization fails are calls that attempt another initialization of
+the context or release the context.
+
 The functions EVP_DigestInit(), EVP_DigestFinal() and EVP_MD_CTX_copy() are
 obsolete but are retained to maintain compatibility with existing code. New
 applications should use EVP_DigestInit_ex(), EVP_DigestFinal_ex() and
@@ -687,10 +695,26 @@ digest name passed on the command line.
      }
 
      mdctx = EVP_MD_CTX_new();
-     EVP_DigestInit_ex2(mdctx, md, NULL);
-     EVP_DigestUpdate(mdctx, mess1, strlen(mess1));
-     EVP_DigestUpdate(mdctx, mess2, strlen(mess2));
-     EVP_DigestFinal_ex(mdctx, md_value, &md_len);
+     if (!EVP_DigestInit_ex2(mdctx, md, NULL)) {
+         printf("Message digest initialization failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestUpdate(mdctx, mess1, strlen(mess1))) {
+         printf("Message digest update failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestUpdate(mdctx, mess2, strlen(mess2))) {
+         printf("Message digest update failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestFinal_ex(mdctx, md_value, &md_len)) {
+         printf("Message digest finalization failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
      EVP_MD_CTX_free(mdctx);
 
      printf("Digest is: ");
diff --git a/doc/man3/EVP_DigestSignInit.pod b/doc/man3/EVP_DigestSignInit.pod
index f142639bfc..d7ed3ec8c0 100644
--- a/doc/man3/EVP_DigestSignInit.pod
+++ b/doc/man3/EVP_DigestSignInit.pod
@@ -172,6 +172,10 @@ multiple times on a context and the parameters set by previous calls should be
 preserved if the I<pkey> parameter is NULL. The call then just resets the state
 of the I<ctx>.
 
+Ignoring failure returns of EVP_DigestSignInit() and EVP_DigestSignInit_ex()
+functions can lead to subsequent undefined behavior when calling
+EVP_DigestSignUpdate(), EVP_DigestSignFinal(), or EVP_DigestSign().
+
 The use of EVP_PKEY_get_size() with these functions is discouraged because some
 signature operations may have a signature length which depends on the
 parameters set. As a result EVP_PKEY_get_size() would have to return a value
diff --git a/doc/man3/EVP_DigestVerifyInit.pod b/doc/man3/EVP_DigestVerifyInit.pod
index 87739a9a02..f67757a1ef 100644
--- a/doc/man3/EVP_DigestVerifyInit.pod
+++ b/doc/man3/EVP_DigestVerifyInit.pod
@@ -161,6 +161,10 @@ multiple times on a context and the parameters set by previous calls should be
 preserved if the I<pkey> parameter is NULL. The call then just resets the state
 of the I<ctx>.
 
+Ignoring failure returns of EVP_DigestVerifyInit() and EVP_DigestVerifyInit_ex()
+functions can lead to subsequent undefined behavior when calling
+EVP_DigestVerifyUpdate(), EVP_DigestVerifyFinal(), or EVP_DigestVerify().
+
 =head1 SEE ALSO
 
 L<EVP_DigestSignInit(3)>,
diff --git a/doc/man3/EVP_EncryptInit.pod b/doc/man3/EVP_EncryptInit.pod
index a96e47f7fb..ca203d3f4a 100644
--- a/doc/man3/EVP_EncryptInit.pod
+++ b/doc/man3/EVP_EncryptInit.pod
@@ -1482,6 +1482,12 @@ removed, and it is especially important for the
 B<EVP_CIPHER_CTX_FLAG_WRAP_ALLOW> flag treated specially in
 EVP_CipherInit_ex().
 
+Ignoring failure returns of the B<EVP_CIPHER_CTX> initialization functions can
+lead to subsequent undefined behavior when calling the functions that update or
+finalize the context. The only valid calls on the B<EVP_CIPHER_CTX> when
+initialization fails are calls that attempt another initialization of the
+context or release the context.
+
 EVP_get_cipherbynid(), and EVP_get_cipherbyobj() are implemented as macros.
 
 =head1 BUGS
@@ -1514,7 +1520,11 @@ Encrypt a string using IDEA:
      FILE *out;
 
      ctx = EVP_CIPHER_CTX_new();
-     EVP_EncryptInit_ex2(ctx, EVP_idea_cbc(), key, iv, NULL);
+     if (!EVP_EncryptInit_ex2(ctx, EVP_idea_cbc(), key, iv, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
 
      if (!EVP_EncryptUpdate(ctx, outbuf, &outlen, intext, strlen(intext))) {
          /* Error */
@@ -1572,13 +1582,21 @@ with a 128-bit key:
 
      /* Don't set key or IV right away; we want to check lengths */
      ctx = EVP_CIPHER_CTX_new();
-     EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
-                        do_encrypt, NULL);
+     if (!EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
+                             do_encrypt, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
      OPENSSL_assert(EVP_CIPHER_CTX_get_key_length(ctx) == 16);
      OPENSSL_assert(EVP_CIPHER_CTX_get_iv_length(ctx) == 16);
 
      /* Now we can set key and IV */
-     EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL);
+     if (!EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
 
      for (;;) {
          inlen = fread(inbuf, 1, 1024, in);