Add support for -1, -2 salt lengths for PSS only keys.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2236)

4 files changed, 25 insertions, 6 deletions

diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index 45fd4caeca..112e5a46ec 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -23,6 +23,7 @@ static ERR_STRING_DATA RSA_str_functs[] = {
     {ERR_FUNC(RSA_F_ENCODE_PKCS1), "encode_pkcs1"},
     {ERR_FUNC(RSA_F_INT_RSA_VERIFY), "int_rsa_verify"},
     {ERR_FUNC(RSA_F_OLD_RSA_PRIV_DECODE), "old_rsa_priv_decode"},
+    {ERR_FUNC(RSA_F_PKEY_PSS_INIT), "pkey_pss_init"},
     {ERR_FUNC(RSA_F_PKEY_RSA_CTRL), "pkey_rsa_ctrl"},
     {ERR_FUNC(RSA_F_PKEY_RSA_CTRL_STR), "pkey_rsa_ctrl_str"},
     {ERR_FUNC(RSA_F_PKEY_RSA_SIGN), "pkey_rsa_sign"},
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index d55fb21c40..c31b9a3cb8 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -432,9 +432,16 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         } else {
             if (p1 < -2)
                 return -2;
-            if (rsa_pss_restricted(rctx) && p1 < rctx->min_saltlen) {
-                RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
-                return 0;
+            if (rsa_pss_restricted(rctx)) {
+                if (p1 == -2 && ctx->operation == EVP_PKEY_OP_VERIFY) {
+                    RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
+                    return -2;
+                }
+                if ((p1 == -1 && rctx->min_saltlen > EVP_MD_size(rctx->md))
+                    || (p1 >= 0 && p1 < rctx->min_saltlen)) {
+                    RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
+                    return 0;
+                }
             }
             rctx->saltlen = p1;
         }
@@ -752,7 +759,7 @@ static int pkey_pss_init(EVP_PKEY_CTX *ctx)
     RSA_PKEY_CTX *rctx = ctx->data;
     const EVP_MD *md;
     const EVP_MD *mgf1md;
-    int min_saltlen;
+    int min_saltlen, max_saltlen;
 
     /* Should never happen */
     if (!pkey_ctx_is_pss(ctx))
@@ -765,6 +772,15 @@ static int pkey_pss_init(EVP_PKEY_CTX *ctx)
     if (!rsa_pss_get_param(rsa->pss, &md, &mgf1md, &min_saltlen))
         return 0;
 
+    /* See if minumum salt length exceeds maximum possible */
+    max_saltlen = RSA_size(rsa) - EVP_MD_size(md);
+    if ((RSA_bits(rsa) & 0x7) == 1)
+        max_saltlen--;
+    if (min_saltlen > max_saltlen) {
+        RSAerr(RSA_F_PKEY_PSS_INIT, RSA_R_INVALID_SALT_LENGTH);
+        return 0;
+    }
+
     rctx->min_saltlen = min_saltlen;
 
     /*
diff --git a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
index eb7dfd8a7b..853d4b8d36 100644
--- a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
+++ b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
@@ -42,9 +42,10 @@ returned if an attempt is made to set the padding mode to anything other
 than B<PSS>. It is otherwise similar to the B<RSA> version.
 
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro is used to set the salt length.
-If the key has usage restrictionsthen an error is returned if an attempt is
+If the key has usage restrictions then an error is returned if an attempt is
 made to set the salt length below the minimum value. It is otherwise similar
-to the B<RSA> operation except special negative values are not supported.
+to the B<RSA> operation except detection of the salt length (using -2) is
+not supported for verification if the key has usage restrictions.
 
 The EVP_PKEY_CTX_set_signature_md() and EVP_PKEY_CTX_set_rsa_mgf1_md() macros
 are used to set the digest and MGF1 algorithms respectively. If the key has
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 95639cb2d7..b9179b36ff 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -476,6 +476,7 @@ int ERR_load_RSA_strings(void);
 # define RSA_F_ENCODE_PKCS1                               146
 # define RSA_F_INT_RSA_VERIFY                             145
 # define RSA_F_OLD_RSA_PRIV_DECODE                        147
+# define RSA_F_PKEY_PSS_INIT                              165
 # define RSA_F_PKEY_RSA_CTRL                              143
 # define RSA_F_PKEY_RSA_CTRL_STR                          144
 # define RSA_F_PKEY_RSA_SIGN                              142