Sign-related fixes (and tests).

BN_mod_exp_mont does not work properly yet if modulus m
is negative (we want computations to be carried out
modulo |m|).

3 files changed, 30 insertions, 17 deletions

diff --git a/crypto/bn/bn_div.c b/crypto/bn/bn_div.c
index 2e600c7c54..64b84ac1a7 100644
--- a/crypto/bn/bn_div.c
+++ b/crypto/bn/bn_div.c
@@ -241,6 +241,8 @@ int BN_div(BIGNUM *dv, BIGNUM *rm, const BIGNUM *num, const BIGNUM *divisor,
 		}
 	else
 		res->top--;
+	if (res->top == 0)
+		res->neg = 0;
 	resp--;
 
 	for (i=0; i<loop-1; i++)
diff --git a/crypto/bn/bn_sqrt.c b/crypto/bn/bn_sqrt.c
index 6959cc5f6f..6e70e5c541 100644
--- a/crypto/bn/bn_sqrt.c
+++ b/crypto/bn/bn_sqrt.c
@@ -133,21 +133,16 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 	e = 1;
 	while (!BN_is_bit_set(p, e))
 		e++;
-	if (e > 2)
-		{
-		/* we don't need this  q  if  e = 1 or 2 */
-		if (!BN_rshift(q, p, e)) goto end;
-		q->neg = 0;
-		}
+	/* we'll set  q  later (if needed) */
 
 	if (e == 1)
 		{
-		/* The easy case:  (p-1)/2  is odd, so 2 has an inverse
-		 * modulo  (p-1)/2,  and square roots can be computed
+		/* The easy case:  (|p|-1)/2  is odd, so 2 has an inverse
+		 * modulo  (|p|-1)/2,  and square roots can be computed
 		 * directly by modular exponentiation.
 		 * We have
-		 *     2 * (p+1)/4 == 1   (mod (p-1)/2),
-		 * so we can use exponent  (p+1)/4,  i.e.  (p-3)/4 + 1.
+		 *     2 * (|p|+1)/4 == 1   (mod (|p|-1)/2),
+		 * so we can use exponent  (|p|+1)/4,  i.e.  (|p|-3)/4 + 1.
 		 */
 		if (!BN_rshift(q, p, 2)) goto end;
 		q->neg = 0;
@@ -159,16 +154,16 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 	
 	if (e == 2)
 		{
-		/* p == 5  (mod 8)
+		/* |p| == 5  (mod 8)
 		 *
 		 * In this case  2  is always a non-square since
 		 * Legendre(2,p) = (-1)^((p^2-1)/8)  for any odd prime.
 		 * So if  a  really is a square, then  2*a  is a non-square.
 		 * Thus for
-		 *      b := (2*a)^((p-5)/8),
+		 *      b := (2*a)^((|p|-5)/8),
 		 *      i := (2*a)*b^2
 		 * we have
-		 *     i^2 = (2*a)^((1 + (p-5)/4)*2)
+		 *     i^2 = (2*a)^((1 + (|p|-5)/4)*2)
 		 *         = (2*a)^((p-1)/2)
 		 *         = -1;
 		 * so if we set
@@ -195,7 +190,7 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 		/* t := 2*a */
 		if (!BN_mod_lshift1_quick(t, a, p)) goto end;
 
-		/* b := (2*a)^((p-5)/8) */
+		/* b := (2*a)^((|p|-5)/8) */
 		if (!BN_rshift(q, p, 3)) goto end;
 		q->neg = 0;
 		if (!BN_mod_exp(b, t, q, p, ctx)) goto end;
@@ -218,6 +213,8 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 	
 	/* e > 2, so we really have to use the Tonelli/Shanks algorithm.
 	 * First, find some  y  that is not a square. */
+	if (!BN_copy(q, p)) goto end; /* use 'q' as temp */
+	q->neg = 0;
 	i = 2;
 	do
 		{
@@ -240,7 +237,7 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 				if (!BN_set_word(y, i)) goto end;
 			}
 		
-		r = BN_kronecker(y, p, ctx);
+		r = BN_kronecker(y, q, ctx); /* here 'q' is |p| */
 		if (r < -1) goto end;
 		if (r == 0)
 			{
@@ -262,6 +259,8 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 		goto end;
 		}
 
+	/* Here's our actual 'q': */
+	if (!BN_rshift(q, q, e)) goto end;
 
 	/* Now that we have some non-square, we can find an element
 	 * of order  2^e  by computing its q'th power. */
diff --git a/crypto/bn/bntest.c b/crypto/bn/bntest.c
index f27087d59c..9f308b75a9 100644
--- a/crypto/bn/bntest.c
+++ b/crypto/bn/bntest.c
@@ -907,6 +907,7 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 	 * works.) */
 
 	if (!BN_generate_prime(b, 512, 0, NULL, NULL, genprime_cb, NULL)) goto err;
+	b->neg = rand_neg();
 	putc('\n', stderr);
 
 	for (i = 0; i < num0; i++)
@@ -914,12 +915,17 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 		if (!BN_bntest_rand(a, 512, 0, 0)) goto err;
 		a->neg = rand_neg();
 
-		/* t := (b-1)/2  (note that b is odd) */
+		/* t := (|b|-1)/2  (note that b is odd) */
 		if (!BN_copy(t, b)) goto err;
+		t->neg = 0;
 		if (!BN_sub_word(t, 1)) goto err;
 		if (!BN_rshift1(t, t)) goto err;
 		/* r := a^t mod b */
-		if (!BN_mod_exp(r, a, t, b, ctx)) goto err;
+		/* FIXME: Using BN_mod_exp (Montgomery variant) leads to
+		 * incorrect results if  b  is negative ("Legendre symbol
+		 * computation failed").
+		 * We want computations to be carried out modulo |b|. */
+		if (!BN_mod_exp_simple(r, a, t, b, ctx)) goto err;
 
 		if (BN_is_word(r, 1))
 			legendre = 1;
@@ -938,6 +944,9 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 		
 		kronecker = BN_kronecker(a, b, ctx);
 		if (kronecker < -1) goto err;
+		/* we actually need BN_kronecker(a, |b|) */
+		if (a->neg && b->neg)
+			kronecker = -kronecker;
 		
 		if (legendre != kronecker)
 			{
@@ -991,6 +1000,7 @@ int test_sqrt(BIO *bp, BN_CTX *ctx)
 			if (!BN_generate_prime(p, 256, 0, a, r, genprime_cb, NULL)) goto err;
 			putc('\n', stderr);
 			}
+		p->neg = rand_neg();
 
 		for (j = 0; j < num2; j++)
 			{
@@ -1003,6 +1013,8 @@ int test_sqrt(BIO *bp, BN_CTX *ctx)
 			if (!BN_nnmod(a, a, p, ctx)) goto err;
 			if (!BN_mod_sqr(a, a, p, ctx)) goto err;
 			if (!BN_mul(a, a, r, ctx)) goto err;
+			if (rand_neg())
+				if (!BN_sub(a, a, p)) goto err;
 
 			if (!BN_mod_sqrt(r, a, p, ctx)) goto err;
 			if (!BN_mod_sqr(r, r, p, ctx)) goto err;