rsa: Add option to disable implicit rejection

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13817)

2 files changed, 30 insertions, 6 deletions

diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 2b25dad893..094a6632b6 100644
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -390,6 +390,12 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     BIGNUM *unblind = NULL;
     BN_BLINDING *blinding = NULL;
 
+    /*
+     * we need the value of the private exponent to perform implicit rejection
+     */
+    if ((rsa->flags & RSA_FLAG_EXT_PKEY) && (padding == RSA_PKCS1_PADDING))
+        padding = RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING;
+
     if ((ctx = BN_CTX_new_ex(rsa->libctx)) == NULL)
         goto err;
     BN_CTX_start(ctx);
@@ -488,7 +494,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
      * derive the Key Derivation Key from private exponent and public
      * ciphertext
      */
-    if (!(rsa->flags & RSA_FLAG_EXT_PKEY)) {
+    if (padding == RSA_PKCS1_PADDING) {
         /*
          * because we use d as a handle to rsa->d we need to keep it local and
          * free before any further use of rsa->d
@@ -564,11 +570,11 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
         goto err;
 
     switch (padding) {
+    case RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING:
+        r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num);
+        break;
     case RSA_PKCS1_PADDING:
-        if (rsa->flags & RSA_FLAG_EXT_PKEY)
-            r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num);
-        else
-            r = ossl_rsa_padding_check_PKCS1_type_2(rsa->libctx, to, num, buf, j, num, kdk);
+        r = ossl_rsa_padding_check_PKCS1_type_2(rsa->libctx, to, num, buf, j, num, kdk);
         break;
     case RSA_PKCS1_OAEP_PADDING:
         r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0);
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 8b35e5c3c6..c67b20baf5 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -52,6 +52,8 @@ typedef struct {
     /* OAEP label */
     unsigned char *oaep_label;
     size_t oaep_labellen;
+    /* if to use implicit rejection in PKCS#1 v1.5 decryption */
+    int implicit_rejection;
 } RSA_PKEY_CTX;
 
 /* True if PSS parameters are restricted */
@@ -72,6 +74,7 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
     /* Maximum for sign, auto for verify */
     rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
     rctx->min_saltlen = -1;
+    rctx->implicit_rejection = 1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
     ctx->keygen_info_count = 2;
@@ -97,6 +100,7 @@ static int pkey_rsa_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src)
     dctx->md = sctx->md;
     dctx->mgf1md = sctx->mgf1md;
     dctx->saltlen = sctx->saltlen;
+    dctx->implicit_rejection = sctx->implicit_rejection;
     if (sctx->oaep_label) {
         OPENSSL_free(dctx->oaep_label);
         dctx->oaep_label = OPENSSL_memdup(sctx->oaep_label, sctx->oaep_labellen);
@@ -345,6 +349,7 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
                             const unsigned char *in, size_t inlen)
 {
     int ret;
+    int pad_mode;
     RSA_PKEY_CTX *rctx = ctx->data;
     /*
      * Discard const. Its marked as const because this may be a cached copy of
@@ -365,7 +370,12 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
                                                 rctx->oaep_labellen,
                                                 rctx->md, rctx->mgf1md);
     } else {
-        ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode);
+        if (rctx->pad_mode == RSA_PKCS1_PADDING &&
+              rctx->implicit_rejection == 0)
+            pad_mode = RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING;
+        else
+            pad_mode = rctx->pad_mode;
+        ret = RSA_private_decrypt(inlen, in, out, rsa, pad_mode);
     }
     *outlen = constant_time_select_s(constant_time_msb_s(ret), *outlen, ret);
     ret = constant_time_select_int(constant_time_msb(ret), ret, 1);
@@ -585,6 +595,14 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         *(unsigned char **)p2 = rctx->oaep_label;
         return rctx->oaep_labellen;
 
+    case EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION:
+        if (rctx->pad_mode != RSA_PKCS1_PADDING) {
+            ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE);
+            return -2;
+        }
+        rctx->implicit_rejection = p1;
+        return 1;
+
     case EVP_PKEY_CTRL_DIGESTINIT:
     case EVP_PKEY_CTRL_PKCS7_SIGN:
 #ifndef OPENSSL_NO_CMS