Add rsa_pss_get_param.

New function rsa_pss_get_param to extract and sanity check PSS parameters.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

3 files changed, 36 insertions, 31 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index c030c27560..671719a3a4 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -596,42 +596,12 @@ static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
     /* Decode PSS parameters */
     pss = rsa_pss_decode(sigalg);
 
-    if (pss == NULL) {
+    if (!rsa_pss_get_param(pss, &md, &mgf1md, &saltlen)) {
         RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_PSS_PARAMETERS);
         goto err;
     }
-    mgf1md = rsa_algor_to_md(pss->maskHash);
-    if (!mgf1md)
-        goto err;
-    md = rsa_algor_to_md(pss->hashAlgorithm);
-    if (!md)
-        goto err;
-
-    if (pss->saltLength) {
-        saltlen = ASN1_INTEGER_get(pss->saltLength);
-
-        /*
-         * Could perform more salt length sanity checks but the main RSA
-         * routines will trap other invalid values anyway.
-         */
-        if (saltlen < 0) {
-            RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_SALT_LENGTH);
-            goto err;
-        }
-    } else
-        saltlen = 20;
-
-    /*
-     * low-level routines support only trailer field 0xbc (value 1) and
-     * PKCS#1 says we should reject any other value anyway.
-     */
-    if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
-        RSAerr(RSA_F_RSA_PSS_TO_CTX, RSA_R_INVALID_TRAILER);
-        goto err;
-    }
 
     /* We have all parameters now set up context */
-
     if (pkey) {
         if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
             goto err;
@@ -661,6 +631,38 @@ static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
     return rv;
 }
 
+int rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
+                      const EVP_MD **pmgf1md, int *psaltlen)
+{
+    if (pss == NULL)
+        return 0;
+    *pmd = rsa_algor_to_md(pss->hashAlgorithm);
+    if (*pmd == NULL)
+        return 0;
+    *pmgf1md = rsa_algor_to_md(pss->maskHash);
+    if (*pmgf1md == NULL)
+        return 0;
+    if (pss->saltLength) {
+        *psaltlen = ASN1_INTEGER_get(pss->saltLength);
+        if (*psaltlen < 0) {
+            RSAerr(RSA_F_RSA_PSS_GET_PARAM, RSA_R_INVALID_SALT_LENGTH);
+            return 0;
+        }
+    } else
+        *psaltlen = 20;
+
+    /*
+     * low-level routines support only trailer field 0xbc (value 1) and
+     * PKCS#1 says we should reject any other value anyway.
+     */
+    if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
+        RSAerr(RSA_F_RSA_PSS_GET_PARAM, RSA_R_INVALID_TRAILER);
+        return 0; 
+    }
+
+    return 1;
+}
+
 #ifndef OPENSSL_NO_CMS
 static int rsa_cms_verify(CMS_SignerInfo *si)
 {
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index 8cc0e6c0a7..ee2ec4d19b 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -77,6 +77,7 @@ static ERR_STRING_DATA RSA_str_functs[] = {
     {ERR_FUNC(RSA_F_RSA_PRINT_FP), "RSA_print_fp"},
     {ERR_FUNC(RSA_F_RSA_PRIV_DECODE), "rsa_priv_decode"},
     {ERR_FUNC(RSA_F_RSA_PRIV_ENCODE), "rsa_priv_encode"},
+    {ERR_FUNC(RSA_F_RSA_PSS_GET_PARAM), "rsa_pss_get_param"},
     {ERR_FUNC(RSA_F_RSA_PSS_TO_CTX), "rsa_pss_to_ctx"},
     {ERR_FUNC(RSA_F_RSA_PUB_DECODE), "rsa_pub_decode"},
     {ERR_FUNC(RSA_F_RSA_SETUP_BLINDING), "RSA_setup_blinding"},
diff --git a/crypto/rsa/rsa_locl.h b/crypto/rsa/rsa_locl.h
index 51916084fe..f2681f96a2 100644
--- a/crypto/rsa/rsa_locl.h
+++ b/crypto/rsa/rsa_locl.h
@@ -103,3 +103,5 @@ extern int int_rsa_verify(int dtype, const unsigned char *m,
 
 RSA_PSS_PARAMS *rsa_pss_params_create(const EVP_MD *sigmd,
                                       const EVP_MD *mgf1md, int saltlen);
+int rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
+                      const EVP_MD **pmgf1md, int *psaltlen);