Limit size of modulus for BN_mod_exp_mont_consttime()

Otherwise the powerbufLen can overflow.

Issue reported by Jiayi Lin.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/19632)

2 files changed, 29 insertions, 0 deletions

diff --git a/crypto/bn/bn_exp.c b/crypto/bn/bn_exp.c
index c7b62232f3..1f6532dc6b 100644
--- a/crypto/bn/bn_exp.c
+++ b/crypto/bn/bn_exp.c
@@ -615,6 +615,15 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
     top = m->top;
 
+    if (in_mont != NULL && BN_is_zero(&in_mont->N)) {
+        ERR_raise(ERR_LIB_BN, ERR_R_PASSED_INVALID_ARGUMENT);
+        return 0;
+    }
+    if ((unsigned int)top > INT_MAX / sizeof(m->d[0]) / (1 << 8)) {
+        /* Prevent overflowing the powerbufLen computation below */
+        ERR_raise(ERR_LIB_BN, BN_R_BIGNUM_TOO_LONG);
+        return 0;
+    }
     /*
      * Use all bits stored in |p|, rather than |BN_num_bits|, so we do not leak
      * whether the top bits are zero.
diff --git a/test/exptest.c b/test/exptest.c
index 7c91e64a58..dc27a5ddc0 100644
--- a/test/exptest.c
+++ b/test/exptest.c
@@ -50,6 +50,7 @@ static int test_mod_exp_zero(void)
     BN_ULONG one_word = 1;
     BN_CTX *ctx = BN_CTX_new();
     int ret = 1, failed = 0;
+    BN_MONT_CTX *mont = NULL;
 
     if (!TEST_ptr(m = BN_new())
         || !TEST_ptr(a = BN_new())
@@ -94,6 +95,24 @@ static int test_mod_exp_zero(void)
     if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
         failed = 1;
 
+    if (!TEST_ptr(mont = BN_MONT_CTX_new()))
+        goto err;
+
+    ERR_set_mark();
+    /* mont is not set but passed in */
+    if (!TEST_false(BN_mod_exp_mont_consttime(r, a, p, m, ctx, mont)))
+        goto err;
+    ERR_pop_to_mark();
+
+    if (!TEST_true(BN_MONT_CTX_set(mont, m, ctx)))
+        goto err;
+
+    if (!TEST_true(BN_mod_exp_mont_consttime(r, a, p, m, ctx, mont)))
+        goto err;
+
+    if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
+        failed = 1;
+
     /*
      * A different codepath exists for single word multiplication
      * in non-constant-time only.
@@ -114,6 +133,7 @@ static int test_mod_exp_zero(void)
     BN_free(a);
     BN_free(p);
     BN_free(m);
+    BN_MONT_CTX_free(mont);
     BN_CTX_free(ctx);
 
     return ret;