From 4514e02cdfc96589d5e8ab0a08942fafa8e418ae Mon Sep 17 00:00:00 2001 From: slontis Date: Mon, 8 Apr 2024 17:12:58 +1000 Subject: Check range of RSA plaintext and ciphertext when using no padding. Fixes #24051 RSA with 'no padding' corresponds to RSAEP/RSADP. The code was not checking the lower bounds. The bounds are specified in SP800-56Br2, section 7.1.1.1 and 7.1.2.1 Note that RFC8017 expresses the range in a sentence using the word between, and there is some ambiguity in this. The upper bounds have change to match the definition in SP800. Reviewed-by: Paul Dale Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/24061) --- crypto/rsa/rsa_ossl.c | 65 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 8 deletions(-) (limited to 'crypto') diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c index 14dfd457f9..5ff67af37d 100644 --- a/crypto/rsa/rsa_ossl.c +++ b/crypto/rsa/rsa_ossl.c @@ -155,10 +155,35 @@ static int rsa_ossl_public_encrypt(int flen, const unsigned char *from, if (BN_bin2bn(buf, num, f) == NULL) goto err; - if (BN_ucmp(f, rsa->n) >= 0) { - /* usually the padding functions would catch this */ - ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); - goto err; +#ifdef FIPS_MODULE + /* + * See SP800-56Br2, section 7.1.1.1 + * RSAEP: 1 < f < (n – 1). + * (where f is the plaintext). + */ + if (padding == RSA_NO_PADDING) { + BIGNUM *nminus1 = BN_CTX_get(ctx); + + if (BN_ucmp(f, BN_value_one()) <= 0) { + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL); + goto err; + } + if (nminus1 == NULL + || BN_copy(nminus1, rsa->n) == NULL + || !BN_sub_word(nminus1, 1)) + goto err; + if (BN_ucmp(f, nminus1) >= 0) { + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); + goto err; + } + } else +#endif + { + if (BN_ucmp(f, rsa->n) >= 0) { + /* usually the padding functions would catch this */ + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); + goto err; + } } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) @@ -546,11 +571,35 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, if (BN_bin2bn(from, (int)flen, f) == NULL) goto err; - if (BN_ucmp(f, rsa->n) >= 0) { - ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); - goto err; - } +#ifdef FIPS_MODULE + /* + * See SP800-56Br2, section 7.1.2.1 + * RSADP: 1 < f < (n – 1) + * (where f is the ciphertext). + */ + if (padding == RSA_NO_PADDING) { + BIGNUM *nminus1 = BN_CTX_get(ctx); + if (BN_ucmp(f, BN_value_one()) <= 0) { + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL); + goto err; + } + if (nminus1 == NULL + || BN_copy(nminus1, rsa->n) == NULL + || !BN_sub_word(nminus1, 1)) + goto err; + if (BN_ucmp(f, nminus1) >= 0) { + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); + goto err; + } + } else +#endif + { + if (BN_ucmp(f, rsa->n) >= 0) { + ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); + goto err; + } + } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, rsa->lock, rsa->n, ctx)) -- cgit v1.2.3