BN_mod_exp(r,a,p,m,ctx) should not be called with r == p.

But even if this is avoided, there are still segmentation violations
(during one of the BN_free()s at the end of test_kron
in some cases, in other cases during BN_kronecker, or
later in BN_sqrt; choosing a different exponentiation
algorithm in bntest.c appears to influence when the SIGSEGV
takes place).

1 files changed, 19 insertions, 45 deletions

diff --git a/crypto/bn/bntest.c b/crypto/bn/bntest.c
index 8289b39dd6..9162999c27 100644
--- a/crypto/bn/bntest.c
+++ b/crypto/bn/bntest.c
@@ -165,6 +165,7 @@ int main(int argc, char *argv[])
 	if (!results)
 		BIO_puts(out,"obase=16\nibase=16\n");
 
+#if 0
 	message(out,"BN_add");
 	if (!test_add(out)) goto err;
 	BIO_flush(out);
@@ -229,6 +230,7 @@ int main(int argc, char *argv[])
 	message(out,"BN_exp");
 	if (!test_exp(out,ctx)) goto err;
 	BIO_flush(out);
+#endif
 
 	message(out,"BN_kronecker");
 	if (!test_kron(out,ctx)) goto err;
@@ -924,7 +926,7 @@ static void genprime_cb(int p, int n, void *arg)
 
 int test_kron(BIO *bp, BN_CTX *ctx)
 	{
-	BIGNUM *a,*b,*r;
+	BIGNUM *a,*b,*r,*t;
 	int i;
 	int legendre, kronecker;
 	int ret = 0;
@@ -932,7 +934,8 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 	a = BN_new();
 	b = BN_new();
 	r = BN_new();
-	if (a == NULL || b == NULL || r == NULL) goto err;
+	t = BN_new();
+	if (a == NULL || b == NULL || r == NULL || t == NULL) goto err;
 	
 	/* We test BN_kronecker(a, b, ctx) just for  b  odd (Jacobi symbol).
 	 * In this case we know that if  b  is prime, then BN_kronecker(a, b, ctx)
@@ -943,7 +946,11 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 	 * don't want to test whether  b  is prime but whether BN_kronecker
 	 * works.) */
 
+#if 0
 	if (!BN_generate_prime(b, 512, 0, NULL, NULL, genprime_cb, NULL)) goto err;
+#else
+	BN_set_word(b,65537);
+#endif
 	putc('\n', stderr);
 
 	for (i = 0; i < num0; i++)
@@ -951,53 +958,19 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 		if (!BN_rand(a, 512, 0, 0)) goto err;
 		a->neg = rand_neg();
 
-		/* r := (b-1)/2  (note that b is odd) */
-		if (!BN_copy(r, b)) goto err;
-		if (!BN_sub_word(r, 1)) goto err;
-		if (!BN_rshift1(r, r)) goto err;
-		/* r := a^r mod b */
-#if 0 /* These three variants should produce the same result, but with
-       * BN_mod_exp_recp or BN_mod_exp_simple, the test fails with
-       * the "Legendre symbol computation failed" error.
-       * (Platform: debug-solaris-sparcv9-gcc)
-       */
-		if (!BN_mod_exp(r, a, r, b, ctx)) goto err;
+		/* t := (b-1)/2  (note that b is odd) */
+		if (!BN_copy(t, b)) goto err;
+		if (!BN_sub_word(t, 1)) goto err;
+		if (!BN_rshift1(t, t)) goto err;
+		/* r := a^t mod b */
+#if 1
+		if (!BN_mod_exp(r, a, t, b, ctx)) goto err;
 #elif 0
-		if (!BN_mod_exp_recp(r, a, r, b, ctx)) goto err;
+		if (!BN_mod_exp_recp(r, a, t, b, ctx)) goto err;
 #else
-		if (!BN_mod_exp_simple(r, a, r, b, ctx)) goto err;
+		if (!BN_mod_exp_simple(r, a, t, b, ctx)) goto err;
 #endif
 
-/*
-On my Linux system, all variants of BN_mod_exp appear to work here,
-but a SIGSEGV occurs later:
-
-Program received signal SIGSEGV, Segmentation fault.
-0x40066e59 in   ()
-(gdb) bt
-#0  0x40066e59 in   ()
-#1  0x40066d3e in   ()
-#2  0x805e64a in CRYPTO_free (str=0x807d968) at mem.c:248
-#3  0x804f68f in bn_expand2 (b=0x807d6b4, words=10) at bn_lib.c:438
-#4  0x8055366 in BN_lshift (r=0x807d6b4, a=0x807d68c, n=63) at bn_shift.c:132
-#5  0x804ca7a in BN_div (dv=0x0, rm=0x807d68c, num=0x807d68c, 
-    divisor=0x807d678, ctx=0x807d610) at bn_div.c:205
-#6  0x805391a in BN_nnmod (r=0x807d68c, m=0x807d68c, d=0x807d678, 
-    ctx=0x807d610) at bn_mod.c:132
-#7  0x8056198 in BN_kronecker (a=0x807d664, b=0x807d848, ctx=0x807d610)
-    at bn_kron.c:170
-#8  0x805d351 in BN_mod_sqrt (in=0x807d860, a=0x807d830, p=0x807d848, 
-    ctx=0x807d610) at bn_sqrt.c:165
-#9  0x804b365 in test_sqrt (bp=0x807d7e8, ctx=0x807d610) at bntest.c:1057
-#10 0x8048da8 in main (argc=0, argv=0xbffffbb8) at bntest.c:240
-#11 0x4002f78a in   ()
-
-These symptoms indicate that the error probably happens earlier
-in the program.  I've disabled the calls to all earlier test_...
-functions and replaced BN_generate_prime by BN_set_word(.., 65537)
-in bntest.c, but this does not help.
- */
-
 		if (BN_is_word(r, 1))
 			legendre = 1;
 		else
@@ -1035,6 +1008,7 @@ in bntest.c, but this does not help.
 	if (a != NULL) BN_free(a);
 	if (b != NULL) BN_free(b);
 	if (r != NULL) BN_free(r);
+	if (t != NULL) BN_free(t);
 	return ret;
 	}