Obtain PSS salt length from provider

Rather than computing the PSS salt length again in core using
ossl_rsa_ctx_to_pss_string, which calls rsa_ctx_to_pss and computes the
salt length, obtain it from the provider using the
OSSL_SIGNATURE_PARAM_ALGORITHM_ID param to handle the case where the
interpretation of the magic constants in the provider differs from that
of OpenSSL core.

Add tests that verify that the rsa_pss_saltlen:max,
rsa_pss_saltlen:<integer> and rsa_pss_saltlen:digest options work and
put the computed digest length into the CMS_ContentInfo struct when
using CMS. Do not add a test for the salt length generated by a provider
when no specific rsa_pss_saltlen option is defined, since that number
could change between providers and provider versions, and we want to
preserve compatibility with older providers.

Signed-off-by: Clemens Lang <cllang@redhat.com>

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19724)

1 files changed, 20 insertions, 18 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 03bbeecee0..08207184ed 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -637,29 +637,31 @@ static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, const void *asn,
     if (pad_mode == RSA_PKCS1_PADDING)
         return 2;
     if (pad_mode == RSA_PKCS1_PSS_PADDING) {
-        ASN1_STRING *os1 = ossl_rsa_ctx_to_pss_string(pkctx);
+        unsigned char aid[128];
+        size_t aid_len = 0;
+        OSSL_PARAM params[2];
 
-        if (os1 == NULL)
+        params[0] = OSSL_PARAM_construct_octet_string(
+            OSSL_SIGNATURE_PARAM_ALGORITHM_ID, aid, sizeof(aid));
+        params[1] = OSSL_PARAM_construct_end();
+
+        if (EVP_PKEY_CTX_get_params(pkctx, params) <= 0)
+            return 0;
+        if ((aid_len = params[0].return_size) == 0)
             return 0;
-        /* Duplicate parameters if we have to */
-        if (alg2 != NULL) {
-            ASN1_STRING *os2 = ASN1_STRING_dup(os1);
 
-            if (os2 == NULL)
-                goto err;
-            if (!X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
-                                 V_ASN1_SEQUENCE, os2)) {
-                ASN1_STRING_free(os2);
-                goto err;
-            }
+        if (alg1 != NULL) {
+            const unsigned char *pp = aid;
+            if (d2i_X509_ALGOR(&alg1, &pp, aid_len) == NULL)
+                return 0;
         }
-        if (!X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
-                             V_ASN1_SEQUENCE, os1))
-            goto err;
+        if (alg2 != NULL) {
+            const unsigned char *pp = aid;
+            if (d2i_X509_ALGOR(&alg2, &pp, aid_len) == NULL)
+                return 0;
+        }
+
         return 3;
-    err:
-        ASN1_STRING_free(os1);
-        return 0;
     }
     return 2;
 }