diff options
| author | Bernd Edlinger <bernd.edlinger@hotmail.de> | 2019-07-04 17:56:23 +0200 |
|---|---|---|
| committer | Bernd Edlinger <bernd.edlinger@hotmail.de> | 2019-07-07 08:07:25 +0200 |
| commit | 9fd44200fe39542c31188de6f3469b438acf39b2 (patch) | |
| tree | a0a833dcb64ab228a9dfaaba8205a38c17614e28 | |
| parent | 78af3f6f95cb8327fb423a609586c3c2b0d9c5f9 (diff) | |
Fix an endless loop in BN_generate_prime_ex
Happens when trying to generate 4 or 5 bit safe primes.
[extended tests]
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9311)
(cherry picked from commit 291f616ced45c924d639d97fc9ca2cbeaad096cf)
| -rw-r--r-- | crypto/bn/bn_prime.c | 8 | ||||
| -rw-r--r-- | test/bntest.c | 47 |
2 files changed, 46 insertions, 9 deletions
diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c index 4bbd7c8810..19b081f38e 100644 --- a/crypto/bn/bn_prime.c +++ b/crypto/bn/bn_prime.c @@ -63,8 +63,12 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, /* There are no prime numbers this small. */ BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL); return 0; - } else if (bits == 2 && safe) { - /* The smallest safe prime (7) is three bits. */ + } else if (add == NULL && safe && bits < 6 && bits != 3) { + /* + * The smallest safe prime (7) is three bits. + * But the following two safe primes with less than 6 bits (11, 23) + * are unreachable for BN_rand with BN_RAND_TOP_TWO. + */ BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL); return 0; } diff --git a/test/bntest.c b/test/bntest.c index c68d7f6fb8..0bd9a06f72 100644 --- a/test/bntest.c +++ b/test/bntest.c @@ -2169,18 +2169,50 @@ static int test_expmodone(void) return ret; } -static int test_smallprime(void) +static int test_smallprime(int kBits) { - static const int kBits = 10; BIGNUM *r; int st = 0; - if (!TEST_ptr(r = BN_new()) - || !TEST_true(BN_generate_prime_ex(r, (int)kBits, 0, - NULL, NULL, NULL)) - || !TEST_int_eq(BN_num_bits(r), kBits)) + if (!TEST_ptr(r = BN_new())) + goto err; + + if (kBits <= 1) { + if (!TEST_false(BN_generate_prime_ex(r, kBits, 0, + NULL, NULL, NULL))) + goto err; + } else { + if (!TEST_true(BN_generate_prime_ex(r, kBits, 0, + NULL, NULL, NULL)) + || !TEST_int_eq(BN_num_bits(r), kBits)) + goto err; + } + + st = 1; + err: + BN_free(r); + return st; +} + +static int test_smallsafeprime(int kBits) +{ + BIGNUM *r; + int st = 0; + + if (!TEST_ptr(r = BN_new())) goto err; + if (kBits <= 5 && kBits != 3) { + if (!TEST_false(BN_generate_prime_ex(r, kBits, 1, + NULL, NULL, NULL))) + goto err; + } else { + if (!TEST_true(BN_generate_prime_ex(r, kBits, 1, + NULL, NULL, NULL)) + || !TEST_int_eq(BN_num_bits(r), kBits)) + goto err; + } + st = 1; err: BN_free(r); @@ -2405,7 +2437,8 @@ int setup_tests(void) ADD_TEST(test_badmod); ADD_TEST(test_expmodzero); ADD_TEST(test_expmodone); - ADD_TEST(test_smallprime); + ADD_ALL_TESTS(test_smallprime, 16); + ADD_ALL_TESTS(test_smallsafeprime, 16); ADD_TEST(test_swap); ADD_TEST(test_ctx_consttime_flag); #ifndef OPENSSL_NO_EC2M |