Initial parameter restrictions.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

3 files changed, 32 insertions, 3 deletions

diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index ee2ec4d19b..749cc6fb1d 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -110,6 +110,7 @@ static ERR_STRING_DATA RSA_str_reasons[] = {
     {ERR_REASON(RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE),
      "data too small for key size"},
     {ERR_REASON(RSA_R_DIGEST_DOES_NOT_MATCH), "digest does not match"},
+    {ERR_REASON(RSA_R_DIGEST_NOT_ALLOWED), "digest not allowed"},
     {ERR_REASON(RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY),
      "digest too big for rsa key"},
     {ERR_REASON(RSA_R_DMP1_NOT_CONGRUENT_TO_D), "dmp1 not congruent to d"},
@@ -135,6 +136,7 @@ static ERR_STRING_DATA RSA_str_reasons[] = {
     {ERR_REASON(RSA_R_IQMP_NOT_INVERSE_OF_Q), "iqmp not inverse of q"},
     {ERR_REASON(RSA_R_KEY_SIZE_TOO_SMALL), "key size too small"},
     {ERR_REASON(RSA_R_LAST_OCTET_INVALID), "last octet invalid"},
+    {ERR_REASON(RSA_R_MGF1_DIGEST_NOT_ALLOWED), "mgf1 digest not allowed"},
     {ERR_REASON(RSA_R_MODULUS_TOO_LARGE), "modulus too large"},
     {ERR_REASON(RSA_R_NO_PUBLIC_EXPONENT), "no public exponent"},
     {ERR_REASON(RSA_R_NULL_BEFORE_BLOCK_MISSING),
@@ -145,6 +147,7 @@ static ERR_STRING_DATA RSA_str_reasons[] = {
      "operation not supported for this keytype"},
     {ERR_REASON(RSA_R_PADDING_CHECK_FAILED), "padding check failed"},
     {ERR_REASON(RSA_R_PKCS_DECODING_ERROR), "pkcs decoding error"},
+    {ERR_REASON(RSA_R_PSS_SALTLEN_TOO_SMALL), "pss saltlen too small"},
     {ERR_REASON(RSA_R_P_NOT_PRIME), "p not prime"},
     {ERR_REASON(RSA_R_Q_NOT_PRIME), "q not prime"},
     {ERR_REASON(RSA_R_RSA_OPERATIONS_NOT_SUPPORTED),
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 80b1e210ef..90e4f07aff 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -35,6 +35,8 @@ typedef struct {
     const EVP_MD *mgf1md;
     /* PSS salt length */
     int saltlen;
+    /* Minimum salt length or -1 if no PSS parameter restriction */
+    int min_saltlen;
     /* Temp buffer */
     unsigned char *tbuf;
     /* OAEP label */
@@ -42,6 +44,9 @@ typedef struct {
     size_t oaep_labellen;
 } RSA_PKEY_CTX;
 
+/* True if PSS parameters are restricted */
+#define rsa_pss_param(rctx) (rctx->min_saltlen != -1)
+
 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 {
     RSA_PKEY_CTX *rctx;
@@ -54,6 +59,7 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
     rctx->saltlen = -2;
+    rctx->min_saltlen = -1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
     ctx->keygen_info_count = 2;
@@ -415,11 +421,15 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
             RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
             return -2;
         }
-        if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
+        if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
             *(int *)p2 = rctx->saltlen;
-        else {
+        } else {
             if (p1 < -2)
                 return -2;
+            if (rsa_pss_param(rctx) && p1 < rctx->min_saltlen) {
+                RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
+                return 0;
+            }
             rctx->saltlen = p1;
         }
         return 1;
@@ -456,6 +466,12 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
     case EVP_PKEY_CTRL_MD:
         if (!check_padding_md(p2, rctx->pad_mode))
             return 0;
+        if (rsa_pss_param(rctx)) {
+            if (EVP_MD_type(rctx->md) == EVP_MD_type(p2))
+                return 1;
+            RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_DIGEST_NOT_ALLOWED);
+            return 0;
+        }
         rctx->md = p2;
         return 1;
 
@@ -475,8 +491,15 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                 *(const EVP_MD **)p2 = rctx->mgf1md;
             else
                 *(const EVP_MD **)p2 = rctx->md;
-        } else
+        } else {
+            if (rsa_pss_param(rctx)) {
+                if (EVP_MD_type(rctx->md) == EVP_MD_type(p2))
+                    return 1;
+                RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_MGF1_DIGEST_NOT_ALLOWED);
+                return 0;
+            }
             rctx->mgf1md = p2;
+        }
         return 1;
 
     case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index a4878d9ccc..08eb8082a8 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -550,6 +550,7 @@ int ERR_load_RSA_strings(void);
 # define RSA_R_DATA_TOO_SMALL                             111
 # define RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE                122
 # define RSA_R_DIGEST_DOES_NOT_MATCH                      158
+# define RSA_R_DIGEST_NOT_ALLOWED                         145
 # define RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY                 112
 # define RSA_R_DMP1_NOT_CONGRUENT_TO_D                    124
 # define RSA_R_DMQ1_NOT_CONGRUENT_TO_D                    125
@@ -573,6 +574,7 @@ int ERR_load_RSA_strings(void);
 # define RSA_R_IQMP_NOT_INVERSE_OF_Q                      126
 # define RSA_R_KEY_SIZE_TOO_SMALL                         120
 # define RSA_R_LAST_OCTET_INVALID                         134
+# define RSA_R_MGF1_DIGEST_NOT_ALLOWED                    152
 # define RSA_R_MODULUS_TOO_LARGE                          105
 # define RSA_R_NO_PUBLIC_EXPONENT                         140
 # define RSA_R_NULL_BEFORE_BLOCK_MISSING                  113
@@ -581,6 +583,7 @@ int ERR_load_RSA_strings(void);
 # define RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE   148
 # define RSA_R_PADDING_CHECK_FAILED                       114
 # define RSA_R_PKCS_DECODING_ERROR                        159
+# define RSA_R_PSS_SALTLEN_TOO_SMALL                      164
 # define RSA_R_P_NOT_PRIME                                128
 # define RSA_R_Q_NOT_PRIME                                129
 # define RSA_R_RSA_OPERATIONS_NOT_SUPPORTED               130