Fix CMS so that it still works with non fetchable algorithms.

Fixes #12633

For CMS the Gost engine still requires calls to EVP_get_digestbyname() and EVP_get_cipherbyname() when
EVP_MD_fetch() and EVP_CIPHER_fetch() return NULL.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/12689)

4 files changed, 58 insertions, 20 deletions

diff --git a/crypto/cms/cms_enc.c b/crypto/cms/cms_enc.c
index e25453ec9c..48934ef2a1 100644
--- a/crypto/cms/cms_enc.c
+++ b/crypto/cms/cms_enc.c
@@ -45,6 +45,7 @@ BIO *cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec,
 
     BIO_get_cipher_ctx(b, &ctx);
 
+    (void)ERR_set_mark();
     if (enc) {
         cipher = ec->cipher;
         /*
@@ -58,17 +59,21 @@ BIO *cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec,
     if (cipher != NULL) {
         fetched_ciph = EVP_CIPHER_fetch(cms_ctx->libctx, EVP_CIPHER_name(cipher),
                                         cms_ctx->propq);
-        if (fetched_ciph == NULL) {
-            CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO, CMS_R_UNKNOWN_CIPHER);
-            goto err;
-        }
+        if (fetched_ciph != NULL)
+            cipher = fetched_ciph;
+    }
+    if (cipher == NULL) {
+        (void)ERR_clear_last_mark();
+        CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO, CMS_R_UNKNOWN_CIPHER);
+        goto err;
     }
-    if (EVP_CipherInit_ex(ctx, fetched_ciph, NULL, NULL, NULL, enc) <= 0) {
+    (void)ERR_pop_to_mark();
+
+    if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, enc) <= 0) {
         CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO,
                CMS_R_CIPHER_INITIALISATION_ERROR);
         goto err;
     }
-    EVP_CIPHER_free(fetched_ciph);
 
     if (enc) {
         int ivlen;
@@ -159,6 +164,7 @@ BIO *cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec,
     ok = 1;
 
  err:
+    EVP_CIPHER_free(fetched_ciph);
     if (!keep_key || !ok) {
         OPENSSL_clear_free(ec->key, ec->keylen);
         ec->key = NULL;
diff --git a/crypto/cms/cms_env.c b/crypto/cms/cms_env.c
index 94961cd038..1fed65c442 100644
--- a/crypto/cms/cms_env.c
+++ b/crypto/cms/cms_env.c
@@ -466,7 +466,8 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms,
     size_t eklen;
     int ret = 0;
     size_t fixlen = 0;
-    EVP_CIPHER *ciph = NULL;
+    const EVP_CIPHER *cipher = NULL;
+    EVP_CIPHER *fetched_cipher = NULL;
     CMS_EncryptedContentInfo *ec;
     const CMS_CTX *ctx = cms_get0_cmsctx(cms);
 
@@ -482,14 +483,22 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms,
         X509_ALGOR *calg = ec->contentEncryptionAlgorithm;
         const char *name = OBJ_nid2sn(OBJ_obj2nid(calg->algorithm));
 
-        ciph = EVP_CIPHER_fetch(ctx->libctx, name, ctx->propq);
-        if (ciph == NULL) {
+        (void)ERR_set_mark();
+        fetched_cipher = EVP_CIPHER_fetch(ctx->libctx, name, ctx->propq);
+
+        if (fetched_cipher != NULL)
+            cipher = fetched_cipher;
+        else
+            cipher = EVP_get_cipherbyobj(calg->algorithm);
+        if (cipher == NULL) {
+            (void)ERR_clear_last_mark();
             CMSerr(CMS_F_CMS_RECIPIENTINFO_KTRI_DECRYPT, CMS_R_UNKNOWN_CIPHER);
             return 0;
         }
+        (void)ERR_pop_to_mark();
 
-        fixlen = EVP_CIPHER_key_length(ciph);
-        EVP_CIPHER_free(ciph);
+        fixlen = EVP_CIPHER_key_length(cipher);
+        EVP_CIPHER_free(fetched_cipher);
     }
 
     ktri->pctx = EVP_PKEY_CTX_new_from_pkey(ctx->libctx, pkey, ctx->propq);
@@ -514,7 +523,6 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms,
         goto err;
 
     ek = OPENSSL_malloc(eklen);
-
     if (ek == NULL) {
         CMSerr(CMS_F_CMS_RECIPIENTINFO_KTRI_DECRYPT, ERR_R_MALLOC_FAILURE);
         goto err;
diff --git a/crypto/cms/cms_lib.c b/crypto/cms/cms_lib.c
index 92321dfc33..7c9b2494a2 100644
--- a/crypto/cms/cms_lib.c
+++ b/crypto/cms/cms_lib.c
@@ -394,26 +394,37 @@ BIO *cms_DigestAlgorithm_init_bio(X509_ALGOR *digestAlgorithm,
 {
     BIO *mdbio = NULL;
     const ASN1_OBJECT *digestoid;
-    EVP_MD *digest = NULL;
+    const EVP_MD *digest = NULL;
+    EVP_MD *fetched_digest = NULL;
     const char *alg;
 
     X509_ALGOR_get0(&digestoid, NULL, NULL, digestAlgorithm);
     alg = OBJ_nid2sn(OBJ_obj2nid(digestoid));
-    digest = EVP_MD_fetch(ctx->libctx, alg, ctx->propq);
+
+    (void)ERR_set_mark();
+    fetched_digest = EVP_MD_fetch(ctx->libctx, alg, ctx->propq);
+
+    if (fetched_digest != NULL)
+        digest = fetched_digest;
+    else
+        digest = EVP_get_digestbyobj(digestoid);
     if (digest == NULL) {
+        (void)ERR_clear_last_mark();
         CMSerr(CMS_F_CMS_DIGESTALGORITHM_INIT_BIO,
                CMS_R_UNKNOWN_DIGEST_ALGORITHM);
         goto err;
     }
+    (void)ERR_pop_to_mark();
+
     mdbio = BIO_new(BIO_f_md());
     if (mdbio == NULL || !BIO_set_md(mdbio, digest)) {
         CMSerr(CMS_F_CMS_DIGESTALGORITHM_INIT_BIO, CMS_R_MD_BIO_INIT_ERROR);
         goto err;
     }
-    EVP_MD_free(digest);
+    EVP_MD_free(fetched_digest);
     return mdbio;
  err:
-    EVP_MD_free(digest);
+    EVP_MD_free(fetched_digest);
     BIO_free(mdbio);
     return NULL;
 }
diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 4fac4e6182..c11d44487b 100644
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -817,7 +817,8 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
     unsigned char *abuf = NULL;
     int alen, r = -1;
     const char *name;
-    EVP_MD *md = NULL;
+    const EVP_MD *md;
+    EVP_MD *fetched_md = NULL;
     const CMS_CTX *ctx = si->cms_ctx;
 
     if (si->pkey == NULL) {
@@ -829,9 +830,21 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
         return -1;
 
     name = OBJ_nid2sn(OBJ_obj2nid(si->digestAlgorithm->algorithm));
-    md = EVP_MD_fetch(ctx->libctx, name, ctx->propq);
-    if (md == NULL)
+
+    (void)ERR_set_mark();
+    fetched_md = EVP_MD_fetch(ctx->libctx, name, ctx->propq);
+
+    if (fetched_md != NULL)
+        md = fetched_md;
+    else
+        md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm);
+    if (md == NULL) {
+        (void)ERR_clear_last_mark();
+        CMSerr(0, CMS_R_UNKNOWN_DIGEST_ALGORITHM);
         return -1;
+    }
+    (void)ERR_pop_to_mark();
+
     if (si->mctx == NULL && (si->mctx = EVP_MD_CTX_new()) == NULL) {
         CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -860,7 +873,7 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
     if (r <= 0)
         CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY, CMS_R_VERIFICATION_FAILURE);
  err:
-    EVP_MD_free(md);
+    EVP_MD_free(fetched_md);
     EVP_MD_CTX_reset(mctx);
     return r;
 }