Defines and strings for special salt length values, add tests

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2236)

3 files changed, 27 insertions, 18 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index ae844eaf1f..20a27be7e2 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -540,7 +540,7 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         saltlen = EVP_MD_size(sigmd);
     else if (saltlen == -2) {
         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
-        if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
+        if ((EVP_PKEY_bits(pk) & 0x7) == 1)
             saltlen--;
     }
 
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index c31b9a3cb8..d4b278ba50 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -58,7 +58,8 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
-    rctx->saltlen = -2;
+    /* Maximum for sign, auto for verify */
+    rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
     rctx->min_saltlen = -1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
@@ -430,14 +431,16 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
             *(int *)p2 = rctx->saltlen;
         } else {
-            if (p1 < -2)
+            if (p1 < RSA_PSS_SALTLEN_MAX)
                 return -2;
             if (rsa_pss_restricted(rctx)) {
-                if (p1 == -2 && ctx->operation == EVP_PKEY_OP_VERIFY) {
+                if (p1 == RSA_PSS_SALTLEN_AUTO
+                    && ctx->operation == EVP_PKEY_OP_VERIFY) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                     return -2;
                 }
-                if ((p1 == -1 && rctx->min_saltlen > EVP_MD_size(rctx->md))
+                if ((p1 == RSA_PSS_SALTLEN_DIGEST
+                     && rctx->min_saltlen > EVP_MD_size(rctx->md))
                     || (p1 >= 0 && p1 < rctx->min_saltlen)) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
                     return 0;
@@ -596,7 +599,14 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 
     if (strcmp(type, "rsa_pss_saltlen") == 0) {
         int saltlen;
-        saltlen = atoi(value);
+        if (!strcmp(value, "digest"))
+            saltlen = RSA_PSS_SALTLEN_DIGEST;
+        else if (!strcmp(value, "max"))
+            saltlen = RSA_PSS_SALTLEN_MAX;
+        else if (!strcmp(value, "auto"))
+            saltlen = RSA_PSS_SALTLEN_AUTO;
+        else
+            saltlen = atoi(value);
         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
     }
 
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 0ec63b2ec7..0a6178b0c4 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -41,7 +41,6 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char H_[EVP_MAX_MD_SIZE];
 
-
     if (ctx == NULL)
         goto err;
 
@@ -57,11 +56,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      *      -2      salt length is autorecovered from signature
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -76,7 +73,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         EM++;
         emLen--;
     }
-    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
+        sLen = emLen - hLen - 2;
+    } else if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
         goto err;
     }
@@ -102,7 +101,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);
         goto err;
     }
-    if (sLen >= 0 && (maskedDBLen - i) != sLen) {
+    if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -160,11 +159,11 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -2      salt length is maximized
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN)
+        sLen = RSA_PSS_SALTLEN_MAX;
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -175,7 +174,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         *EM++ = 0;
         emLen--;
     }
-    if (sLen == -2) {
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
     } else if (emLen < (hLen + sLen + 2)) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,