diff options
| author | Bernd Edlinger <bernd.edlinger@hotmail.de> | 2019-07-04 17:56:23 +0200 |
|---|---|---|
| committer | Bernd Edlinger <bernd.edlinger@hotmail.de> | 2019-07-07 08:05:33 +0200 |
| commit | 291f616ced45c924d639d97fc9ca2cbeaad096cf (patch) | |
| tree | 86fbc3f0f65398c02bbad3f975f5d36bd69c2a06 | |
| parent | 2a1e2fe145c6eb8e75aa2e1b3a8c3a49384b2852 (diff) | |
Fix an endless loop in BN_generate_prime_ex
Happens when trying to generate 4 or 5 bit safe primes.
[extended tests]
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9311)
| -rw-r--r-- | crypto/bn/bn_prime.c | 8 | ||||
| -rw-r--r-- | test/bntest.c | 47 |
2 files changed, 46 insertions, 9 deletions
diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c index 03402c2cec..47e2f2357a 100644 --- a/crypto/bn/bn_prime.c +++ b/crypto/bn/bn_prime.c @@ -98,8 +98,12 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, /* There are no prime numbers this small. */ BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL); return 0; - } else if (bits == 2 && safe) { - /* The smallest safe prime (7) is three bits. */ + } else if (add == NULL && safe && bits < 6 && bits != 3) { + /* + * The smallest safe prime (7) is three bits. + * But the following two safe primes with less than 6 bits (11, 23) + * are unreachable for BN_rand with BN_RAND_TOP_TWO. + */ BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL); return 0; } diff --git a/test/bntest.c b/test/bntest.c index 8df6e0f34c..1e502103c0 100644 --- a/test/bntest.c +++ b/test/bntest.c @@ -2249,18 +2249,50 @@ static int test_expmodone(void) return ret; } -static int test_smallprime(void) +static int test_smallprime(int kBits) { - static const int kBits = 10; BIGNUM *r; int st = 0; - if (!TEST_ptr(r = BN_new()) - || !TEST_true(BN_generate_prime_ex(r, (int)kBits, 0, - NULL, NULL, NULL)) - || !TEST_int_eq(BN_num_bits(r), kBits)) + if (!TEST_ptr(r = BN_new())) + goto err; + + if (kBits <= 1) { + if (!TEST_false(BN_generate_prime_ex(r, kBits, 0, + NULL, NULL, NULL))) + goto err; + } else { + if (!TEST_true(BN_generate_prime_ex(r, kBits, 0, + NULL, NULL, NULL)) + || !TEST_int_eq(BN_num_bits(r), kBits)) + goto err; + } + + st = 1; + err: + BN_free(r); + return st; +} + +static int test_smallsafeprime(int kBits) +{ + BIGNUM *r; + int st = 0; + + if (!TEST_ptr(r = BN_new())) goto err; + if (kBits <= 5 && kBits != 3) { + if (!TEST_false(BN_generate_prime_ex(r, kBits, 1, + NULL, NULL, NULL))) + goto err; + } else { + if (!TEST_true(BN_generate_prime_ex(r, kBits, 1, + NULL, NULL, NULL)) + || !TEST_int_eq(BN_num_bits(r), kBits)) + goto err; + } + st = 1; err: BN_free(r); @@ -2518,7 +2550,8 @@ int setup_tests(void) ADD_TEST(test_badmod); ADD_TEST(test_expmodzero); ADD_TEST(test_expmodone); - ADD_TEST(test_smallprime); + ADD_ALL_TESTS(test_smallprime, 16); + ADD_ALL_TESTS(test_smallsafeprime, 16); ADD_TEST(test_swap); ADD_TEST(test_ctx_consttime_flag); #ifndef OPENSSL_NO_EC2M |