Limit size of modulus for BN_mod_exp_mont_consttime()

Otherwise the powerbufLen can overflow.

Issue reported by Jiayi Lin.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/19632)

(cherry picked from commit 4378e3cd2a4d73a97a2349efaa143059d8ed05e8)

1 files changed, 20 insertions, 0 deletions

diff --git a/test/exptest.c b/test/exptest.c
index 234ad7b2b9..f0d8693130 100644
--- a/test/exptest.c
+++ b/test/exptest.c
@@ -50,6 +50,7 @@ static int test_mod_exp_zero(void)
     BN_ULONG one_word = 1;
     BN_CTX *ctx = BN_CTX_new();
     int ret = 1, failed = 0;
+    BN_MONT_CTX *mont = NULL;
 
     if (!TEST_ptr(m = BN_new())
         || !TEST_ptr(a = BN_new())
@@ -94,6 +95,24 @@ static int test_mod_exp_zero(void)
     if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
         failed = 1;
 
+    if (!TEST_ptr(mont = BN_MONT_CTX_new()))
+        goto err;
+
+    ERR_set_mark();
+    /* mont is not set but passed in */
+    if (!TEST_false(BN_mod_exp_mont_consttime(r, a, p, m, ctx, mont)))
+        goto err;
+    ERR_pop_to_mark();
+
+    if (!TEST_true(BN_MONT_CTX_set(mont, m, ctx)))
+        goto err;
+
+    if (!TEST_true(BN_mod_exp_mont_consttime(r, a, p, m, ctx, mont)))
+        goto err;
+
+    if (!TEST_true(a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a)))
+        failed = 1;
+
     /*
      * A different codepath exists for single word multiplication
      * in non-constant-time only.
@@ -114,6 +133,7 @@ static int test_mod_exp_zero(void)
     BN_free(a);
     BN_free(p);
     BN_free(m);
+    BN_MONT_CTX_free(mont);
     BN_CTX_free(ctx);
 
     return ret;