Make DSA_generate_parameters, and fix a couple of bug

(including another problem in the s3_srvr.c state machine).

4 files changed, 56 insertions, 100 deletions

diff --git a/crypto/dsa/dsa.h b/crypto/dsa/dsa.h
index 3da104b6dd..e59c7d293f 100644
--- a/crypto/dsa/dsa.h
+++ b/crypto/dsa/dsa.h
@@ -197,7 +197,11 @@ int	DSAparams_print_fp(FILE *fp, DSA *x);
 int	DSA_print_fp(FILE *bp, DSA *x, int off);
 #endif
 
-int DSA_is_prime(BIGNUM *q,void (*callback)(),void *cb_arg);
+#define DSS_prime_checks 50
+/* Primality test according to FIPS PUB 186[-1], Appendix 2.1:
+ * 50 rounds of Rabin-Miller */
+#define DSA_is_prime(n, callback, cb_arg) \
+	BN_is_prime(n, DSS_prime_checks, callback, NULL, cb_arg)
 
 #ifndef NO_DH
 /* Convert DSA structure (key or just parameters) into DH structure
@@ -218,7 +222,6 @@ DH *DSA_dup_DH(DSA *r);
 #define DSA_F_DSAPARAMS_PRINT_FP			 101
 #define DSA_F_DSA_DO_SIGN				 112
 #define DSA_F_DSA_DO_VERIFY				 113
-#define DSA_F_DSA_IS_PRIME				 102
 #define DSA_F_DSA_NEW					 103
 #define DSA_F_DSA_PRINT					 104
 #define DSA_F_DSA_PRINT_FP				 105
diff --git a/crypto/dsa/dsa_err.c b/crypto/dsa/dsa_err.c
index 33a8270afd..38e4af968c 100644
--- a/crypto/dsa/dsa_err.c
+++ b/crypto/dsa/dsa_err.c
@@ -70,7 +70,6 @@ static ERR_STRING_DATA DSA_str_functs[]=
 {ERR_PACK(0,DSA_F_DSAPARAMS_PRINT_FP,0),	"DSAparams_print_fp"},
 {ERR_PACK(0,DSA_F_DSA_DO_SIGN,0),	"DSA_do_sign"},
 {ERR_PACK(0,DSA_F_DSA_DO_VERIFY,0),	"DSA_do_verify"},
-{ERR_PACK(0,DSA_F_DSA_IS_PRIME,0),	"DSA_is_prime"},
 {ERR_PACK(0,DSA_F_DSA_NEW,0),	"DSA_new"},
 {ERR_PACK(0,DSA_F_DSA_PRINT,0),	"DSA_print"},
 {ERR_PACK(0,DSA_F_DSA_PRINT_FP,0),	"DSA_print_fp"},
diff --git a/crypto/dsa/dsa_gen.c b/crypto/dsa/dsa_gen.c
index 5df9132dd8..f7e0b585a1 100644
--- a/crypto/dsa/dsa_gen.c
+++ b/crypto/dsa/dsa_gen.c
@@ -59,12 +59,18 @@
 #undef GENUINE_DSA
 
 #ifdef GENUINE_DSA
+/* Parameter generation follows the original release of FIPS PUB 186,
+ * Appendix 2.2 (i.e. use SHA as defined in FIPS PUB 180) */
 #define HASH    SHA
 #else
+/* Parameter generation follows the updated Appendix 2.2 for FIPS PUB 186,
+ * also Appendix 2.2 of FIPS PUB 186-1 (i.e. use SHA as defined in
+ * FIPS PUB 180-1) */
 #define HASH    SHA1
 #endif 
 
 #ifndef NO_SHA
+
 #include <stdio.h>
 #include <time.h>
 #include "cryptlib.h"
@@ -74,8 +80,8 @@
 #include <openssl/rand.h>
 
 DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
-	     int *counter_ret, unsigned long *h_ret, void (*callback)(),
-	     void *cb_arg)
+		int *counter_ret, unsigned long *h_ret, void (*callback)(),
+		void *cb_arg)
 	{
 	int ok=0;
 	unsigned char seed[SHA_DIGEST_LENGTH];
@@ -86,19 +92,26 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 	BN_MONT_CTX *mont=NULL;
 	int k,n=0,i,b,m=0;
 	int counter=0;
-	BN_CTX *ctx=NULL,*ctx2=NULL;
+	int r=0;
+	BN_CTX *ctx=NULL,*ctx2=NULL,*ctx3=NULL,*ctx4=NULL;
 	unsigned int h=2;
 	DSA *ret=NULL;
 
 	if (bits < 512) bits=512;
 	bits=(bits+63)/64*64;
 
-	if (seed_len < 20) seed_in = NULL;
+	if (seed_len < 20)
+		seed_in = NULL; /* seed buffer too small -- ignore */
+	if (seed_len > 20) 
+		seed_len = 20; /* App. 2.2 of FIPS PUB 186 allows larger SEED,
+		                * but our internal buffers are restricted to 160 bits*/
 	if ((seed_in != NULL) && (seed_len == 20))
 		memcpy(seed,seed_in,seed_len);
 
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
 	if ((ctx2=BN_CTX_new()) == NULL) goto err;
+	if ((ctx3=BN_CTX_new()) == NULL) goto err;
+	if ((ctx4=BN_CTX_new()) == NULL) goto err;
 	if ((ret=DSA_new()) == NULL) goto err;
 
 	if ((mont=BN_MONT_CTX_new()) == NULL) goto err;
@@ -116,18 +129,24 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 
 	for (;;)
 		{
-		for (;;)
+		for (;;) /* find q */
 			{
+			int seed_is_random = 0;
+
 			/* step 1 */
 			if (callback != NULL) callback(0,m++,cb_arg);
 
 			if (!seed_len)
+				{
 				RAND_pseudo_bytes(seed,SHA_DIGEST_LENGTH);
+				seed_is_random = 1;
+				}
 			else
+				/* use random seed if 'seed_in' turns out to be bad */
 				seed_len=0;
-
 			memcpy(buf,seed,SHA_DIGEST_LENGTH);
 			memcpy(buf2,seed,SHA_DIGEST_LENGTH);
+			/* precompute "SEED + 1" for step 7: */
 			for (i=SHA_DIGEST_LENGTH-1; i >= 0; i--)
 				{
 				buf[i]++;
@@ -146,7 +165,12 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 			if (!BN_bin2bn(md,SHA_DIGEST_LENGTH,q)) goto err;
 
 			/* step 4 */
-			if (BN_is_prime(q,BN_prime_checks,callback,NULL,cb_arg) > 0) break;
+			r = BN_is_prime_fasttest(q, DSS_prime_checks, callback, ctx3, ctx4, cb_arg, seed_is_random);
+			if (r > 0)
+					break;
+			if (r != 0)
+					goto err;
+
 			/* do a callback call */
 			/* step 5 */
 			}
@@ -156,16 +180,22 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 
 		/* step 6 */
 		counter=0;
+		/* "offset = 2" */
 
 		n=(bits-1)/160;
 		b=(bits-1)-n*160;
 
 		for (;;)
 			{
+			if (callback != NULL && counter != 0)
+				callback(0,counter,cb_arg);
+
 			/* step 7 */
 			BN_zero(W);
+			/* now 'buf' contains "SEED + offset - 1" */
 			for (k=0; k<=n; k++)
 				{
+				/* obtain "SEED + offset + k" by incrementing: */
 				for (i=SHA_DIGEST_LENGTH-1; i >= 0; i--)
 					{
 					buf[i]++;
@@ -196,17 +226,19 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 			if (BN_cmp(p,test) >= 0)
 				{
 				/* step 11 */
-				if (BN_is_prime(p,BN_prime_checks,callback,NULL,cb_arg) > 0)
-					goto end;
+				r = BN_is_prime_fasttest(p, DSS_prime_checks, callback, ctx3, ctx4, cb_arg, 1);
+				if (r > 0)
+						goto end; /* found it */
+				if (r != 0)
+					goto err;
 				}
 
 			/* step 13 */
 			counter++;
+			/* "offset = offset + n + 1" */
 
 			/* step 14 */
 			if (counter >= 4096) break;
-
-			if (callback != NULL) callback(0,counter,cb_arg);
 			}
 		}
 end:
@@ -247,90 +279,10 @@ err:
 		if (h_ret != NULL) *h_ret=h;
 		}
 	if (ctx != NULL) BN_CTX_free(ctx);
-	if (ctx != NULL) BN_CTX_free(ctx2);
+	if (ctx2 != NULL) BN_CTX_free(ctx2);
+	if (ctx3 != NULL) BN_CTX_free(ctx3);
+	if (ctx4 != NULL) BN_CTX_free(ctx4);
 	if (mont != NULL) BN_MONT_CTX_free(mont);
 	return(ok?ret:NULL);
 	}
-
-int DSA_is_prime(BIGNUM *w, void (*callback)(), void *cb_arg)
-	{
-	int ok= -1,j,i,n;
-	BN_CTX *ctx=NULL,*ctx2=NULL;
-	BIGNUM *w_1,*b,*m,*z,*tmp,*mont_1;
-	int a;
-	BN_MONT_CTX *mont=NULL;
-
-	if (!BN_is_odd(w)) return(0);
-
-	if ((ctx=BN_CTX_new()) == NULL) goto err;
-	if ((ctx2=BN_CTX_new()) == NULL) goto err;
-	if ((mont=BN_MONT_CTX_new()) == NULL) goto err;
-
-	m=   &(ctx2->bn[2]);
-	b=   &(ctx2->bn[3]);
-	z=   &(ctx2->bn[4]);
-	w_1= &(ctx2->bn[5]);
-	tmp= &(ctx2->bn[6]);
-	mont_1= &(ctx2->bn[7]);
-
-	/* step 1 */
-	n=BN_prime_checks_size(BN_num_bits(w));
-
-	/* step 2 */
-	if (!BN_sub(w_1,w,BN_value_one())) goto err;
-	for (a=1; !BN_is_bit_set(w_1,a); a++)
-		;
-	if (!BN_rshift(m,w_1,a)) goto err;
-
-	BN_MONT_CTX_set(mont,w,ctx);
-	BN_to_montgomery(mont_1,BN_value_one(),mont,ctx);
-	BN_to_montgomery(w_1,w_1,mont,ctx);
-	for (i=1; i < n; i++)
-		{
-		/* step 3 */
-		if (!BN_pseudo_rand(b,BN_num_bits(w)-2/*-1*/,0,0))
-			goto err;
-		/* BN_set_word(b,0x10001L); */
-
-		/* step 4 */
-		j=0;
-		if (!BN_mod_exp_mont(z,b,m,w,ctx,mont)) goto err;
-
-		if (!BN_to_montgomery(z,z,mont,ctx)) goto err;
-
-		/* step 5 */
-		for (;;)
-			{
-			if (((j == 0) && (BN_cmp(z,mont_1) == 0)) ||
-				(BN_cmp(z,w_1) == 0))
-				break;
-
-			/* step 6 */
-			if ((j > 0) && (BN_cmp(z,mont_1) == 0))
-				{
-				ok=0;
-				goto err;
-				}
-
-			j++;
-			if (j >= a)
-				{
-				ok=0;
-				goto err;
-				}
-
-			if (!BN_mod_mul_montgomery(z,z,z,mont,ctx)) goto err;
-			if (callback != NULL) callback(1,j,cb_arg);
-			}
-		}
-
-	ok=1;
-err:
-	if (ok == -1) DSAerr(DSA_F_DSA_IS_PRIME,ERR_R_BN_LIB);
-	BN_CTX_free(ctx);
-	BN_CTX_free(ctx2);
-	BN_MONT_CTX_free(mont);
-	
-	return(ok);
-	}
 #endif
diff --git a/crypto/dsa/dsatest.c b/crypto/dsa/dsatest.c
index a30dae6b72..f096ed0688 100644
--- a/crypto/dsa/dsatest.c
+++ b/crypto/dsa/dsatest.c
@@ -85,6 +85,9 @@ int main(int argc, char *argv[])
 #endif
 
 static void MS_CALLBACK dsa_cb(int p, int n, char *arg);
+
+/* seed, out_p, out_q, out_g are taken from the updated Appendix 5 to
+ * FIPS PUB 186 and also appear in Appendix 5 to FIPS PIB 186-1 */
 static unsigned char seed[20]={
 	0xd5,0x01,0x4e,0x4b,0x60,0xef,0x2b,0xa8,0xb6,0x21,0x1b,0x40,
 	0x62,0xba,0x32,0x24,0xe0,0x42,0x7d,0xd3,
@@ -141,9 +144,8 @@ int main(int argc, char **argv)
 	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
 	BIO_printf(bio_err,"test generation of DSA parameters\n");
-	BIO_printf(bio_err,"expect '.*' followed by 3 lines of '.'s and '+'s\n");
-	dsa=DSA_generate_parameters(512,seed,20,&counter,&h,dsa_cb,
-		(char *)bio_err);
+
+	dsa=DSA_generate_parameters(512,seed,20,&counter,&h,dsa_cb,(char *)bio_err);
 
 	BIO_printf(bio_err,"seed\n");
 	for (i=0; i<20; i+=4)