BN_mod_exp_mont_consttime: check for zero modulus.

Don't dereference |d| when |top| is zero. Also test that various BIGNUM methods behave correctly on zero/even inputs.

Follow-up to b11980d79a52ec08844f08bea0e66c04b691840b

Reviewed-by: Rich Salz <rsalz@openssl.org>

2 files changed, 56 insertions, 3 deletions

diff --git a/crypto/bn/bn_exp.c b/crypto/bn/bn_exp.c
index 10dc3eb358..66feddcf96 100644
--- a/crypto/bn/bn_exp.c
+++ b/crypto/bn/bn_exp.c
@@ -662,12 +662,13 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     bn_check_top(p);
     bn_check_top(m);
 
-    top = m->top;
-
-    if (!(m->d[0] & 1)) {
+    if (!BN_is_odd(m)) {
         BNerr(BN_F_BN_MOD_EXP_MONT_CONSTTIME, BN_R_CALLED_WITH_EVEN_MODULUS);
         return (0);
     }
+
+    top = m->top;
+
     bits = BN_num_bits(p);
     if (bits == 0) {
         ret = BN_one(rr);
diff --git a/test/bntest.c b/test/bntest.c
index 430d2a02b8..effbd75246 100644
--- a/test/bntest.c
+++ b/test/bntest.c
@@ -451,6 +451,14 @@ int test_div(BIO *bp, BN_CTX *ctx)
     d = BN_new();
     e = BN_new();
 
+    BN_one(a);
+    BN_zero(b);
+
+    if (BN_div(d, c, a, b, ctx)) {
+        fprintf(stderr, "Division by zero succeeded!\n");
+        return 0;
+    }
+
     for (i = 0; i < num0 + num1; i++) {
         if (i < num1) {
             BN_bntest_rand(a, 400, 0, 0);
@@ -787,6 +795,18 @@ int test_mont(BIO *bp, BN_CTX *ctx)
     if (mont == NULL)
         return 0;
 
+    BN_zero(n);
+    if (BN_MONT_CTX_set(mont, n, ctx)) {
+        fprintf(stderr, "BN_MONT_CTX_set succeeded for zero modulus!\n");
+        return 0;
+    }
+
+    BN_set_word(n, 16);
+    if (BN_MONT_CTX_set(mont, n, ctx)) {
+        fprintf(stderr, "BN_MONT_CTX_set succeeded for even modulus!\n");
+        return 0;
+    }
+
     BN_bntest_rand(a, 100, 0, 0);
     BN_bntest_rand(b, 100, 0, 0);
     for (i = 0; i < num2; i++) {
@@ -888,6 +908,14 @@ int test_mod_mul(BIO *bp, BN_CTX *ctx)
     d = BN_new();
     e = BN_new();
 
+    BN_one(a);
+    BN_one(b);
+    BN_zero(c);
+    if (BN_mod_mul(e, a, b, c, ctx)) {
+        fprintf(stderr, "BN_mod_mul with zero modulus succeeded!\n");
+        return 0;
+    }
+
     for (j = 0; j < 3; j++) {
         BN_bntest_rand(c, 1024, 0, 0);
         for (i = 0; i < num0; i++) {
@@ -953,6 +981,14 @@ int test_mod_exp(BIO *bp, BN_CTX *ctx)
     d = BN_new();
     e = BN_new();
 
+    BN_one(a);
+    BN_one(b);
+    BN_zero(c);
+    if (BN_mod_exp(d, a, b, c, ctx)) {
+        fprintf(stderr, "BN_mod_exp with zero modulus succeeded!\n");
+        return 0;
+    }
+
     BN_bntest_rand(c, 30, 0, 1); /* must be odd for montgomery */
     for (i = 0; i < num2; i++) {
         BN_bntest_rand(a, 20 + i * 5, 0, 0);
@@ -1000,6 +1036,22 @@ int test_mod_exp_mont_consttime(BIO *bp, BN_CTX *ctx)
     d = BN_new();
     e = BN_new();
 
+    BN_one(a);
+    BN_one(b);
+    BN_zero(c);
+    if (BN_mod_exp_mont_consttime(d, a, b, c, ctx, NULL)) {
+        fprintf(stderr, "BN_mod_exp_mont_consttime with zero modulus "
+                "succeeded\n");
+        return 0;
+    }
+
+    BN_set_word(c, 16);
+    if (BN_mod_exp_mont_consttime(d, a, b, c, ctx, NULL)) {
+        fprintf(stderr, "BN_mod_exp_mont_consttime with even modulus "
+                "succeeded\n");
+        return 0;
+    }
+
     BN_bntest_rand(c, 30, 0, 1); /* must be odd for montgomery */
     for (i = 0; i < num2; i++) {
         BN_bntest_rand(a, 20 + i * 5, 0, 0);