Update RSA keygen to use sp800-56b by default

Fixes #11742
Fixes #11764

The newer RSA sp800-56b algorithm is being used for the normal case of a non multiprime key of at least length 2048.
Insecure key lengths and mutltiprime RSA will use the old method.

Bad public exponents are no longer allowed (i.e values less than 65537 or even). Values such as 2 that would cause a infinite loop now result in an error. The value of 3 has been marked as deprecated but is still allowed for legacy purposes.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11765)

3 files changed, 54 insertions, 13 deletions

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 5712aa1791..e52bce6355 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -70,16 +70,10 @@ int RSA_generate_multi_prime_key(RSA *rsa, int bits, int primes,
     return rsa_keygen(NULL, rsa, bits, primes, e_value, cb, 0);
 }
 
-static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
-                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+#ifndef FIPS_MODULE
+static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes,
+                                 BIGNUM *e_value, BN_GENCB *cb)
 {
-    int ok = -1;
-#ifdef FIPS_MODULE
-    if (primes != 2)
-        return 0;
-    ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
-    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
-#else
     BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *prime;
     int n = 0, bitsr[RSA_MAX_PRIME_NUM], bitse = 0;
     int i = 0, quo = 0, rmd = 0, adj = 0, retries = 0;
@@ -88,6 +82,7 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     BN_CTX *ctx = NULL;
     BN_ULONG bitst = 0;
     unsigned long error = 0;
+    int ok = -1;
 
     if (bits < RSA_MIN_MODULUS_BITS) {
         ok = 0;             /* we set our own err */
@@ -95,6 +90,12 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
         goto err;
     }
 
+    /* A bad value for e can cause infinite loops */
+    if (e_value != NULL && !rsa_check_public_exponent(e_value)) {
+        RSAerr(0, RSA_R_PUB_EXPONENT_OUT_OF_RANGE);
+        return 0;
+    }
+
     if (primes < RSA_DEFAULT_PRIME_NUM || primes > rsa_multip_cap(bits)) {
         ok = 0;             /* we set our own err */
         RSAerr(0, RSA_R_KEY_PRIME_NUM_INVALID);
@@ -407,8 +408,29 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     }
     BN_CTX_end(ctx);
     BN_CTX_free(ctx);
+    return ok;
+}
 #endif /* FIPS_MODULE */
 
+static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
+                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+{
+    int ok = 0;
+
+    /*
+     * Only multi-prime keys or insecure keys with a small key length will use
+     * the older rsa_multiprime_keygen().
+     */
+    if (primes == 2 && bits >= 2048)
+        ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
+#ifndef FIPS_MODULE
+    else
+        ok = rsa_multiprime_keygen(rsa, bits, primes, e_value, cb);
+#endif /* FIPS_MODULE */
+
+#ifdef FIPS_MODULE
+    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
+#endif
     if (pairwise_test && ok > 0) {
         OSSL_CALLBACK *stcb = NULL;
         void *stcbarg = NULL;
diff --git a/crypto/rsa/rsa_sp800_56b_check.c b/crypto/rsa/rsa_sp800_56b_check.c
index 7cf6241dee..df9b7bf054 100644
--- a/crypto/rsa/rsa_sp800_56b_check.c
+++ b/crypto/rsa/rsa_sp800_56b_check.c
@@ -189,12 +189,30 @@ int rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)
     return ret;
 }
 
+#ifndef FIPS_MODULE
+static int bn_is_three(const BIGNUM *bn)
+{
+    BIGNUM *num = BN_dup(bn);
+    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
+
+    BN_free(num);
+    return ret;
+}
+#endif /* FIPS_MODULE */
+
 /* Check exponent is odd, and has a bitlen ranging from [17..256] */
 int rsa_check_public_exponent(const BIGNUM *e)
 {
-    int bitlen = BN_num_bits(e);
+    int bitlen;
+
+    /* For legacy purposes RSA_3 is allowed in non fips mode */
+#ifndef FIPS_MODULE
+    if (bn_is_three(e))
+        return 1;
+#endif /* FIPS_MODULE */
 
-    return (BN_is_odd(e) &&  bitlen > 16 && bitlen < 257);
+    bitlen = BN_num_bits(e);
+    return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
 }
 
 /*
diff --git a/crypto/rsa/rsa_sp800_56b_gen.c b/crypto/rsa/rsa_sp800_56b_gen.c
index c4c7c08e94..d1673d5c98 100644
--- a/crypto/rsa/rsa_sp800_56b_gen.c
+++ b/crypto/rsa/rsa_sp800_56b_gen.c
@@ -65,7 +65,7 @@ int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,
      * Signature Generation and Key Agree/Transport.
      */
     if (nbits < RSA_FIPS1864_MIN_KEYGEN_KEYSIZE) {
-        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_INVALID_KEY_LENGTH);
+        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_KEY_SIZE_TOO_SMALL);
         return 0;
     }
 
@@ -146,12 +146,13 @@ err:
 int rsa_sp800_56b_validate_strength(int nbits, int strength)
 {
     int s = (int)ifc_ffc_compute_security_bits(nbits);
-
+#ifdef FIPS_MODULE
     if (s < RSA_FIPS1864_MIN_KEYGEN_STRENGTH
             || s > RSA_FIPS1864_MAX_KEYGEN_STRENGTH) {
         RSAerr(RSA_F_RSA_SP800_56B_VALIDATE_STRENGTH, RSA_R_INVALID_MODULUS);
         return 0;
     }
+#endif
     if (strength != -1 && s != strength) {
         RSAerr(RSA_F_RSA_SP800_56B_VALIDATE_STRENGTH, RSA_R_INVALID_STRENGTH);
         return 0;