This is a first-cut at improving the callback mechanisms used in

key-generation and prime-checking functions. Rather than explicitly passing
callback functions and caller-defined context data for the callbacks, a new
structure BN_GENCB is defined that encapsulates this; a pointer to the
structure is passed to all such functions instead.

This wrapper structure allows the encapsulation of "old" and "new" style
callbacks - "new" callbacks return a boolean result on the understanding
that returning FALSE should terminate keygen/primality processing.  The
BN_GENCB abstraction will allow future callback modifications without
needing to break binary compatibility nor change the API function
prototypes. The new API functions have been given names ending in "_ex" and
the old functions are implemented as wrappers to the new ones.  The
OPENSSL_NO_DEPRECATED symbol has been introduced so that, if defined,
declaration of the older functions will be skipped. NB: Some
openssl-internal code will stick with the older callbacks for now, so
appropriate "#undef" logic will be put in place - this is in case the user
is *building* openssl (rather than *including* its headers) with this
symbol defined.

There is another change in the new _ex functions; the key-generation
functions do not return key structures but operate on structures passed by
the caller, the return value is a boolean. This will allow for a smoother
transition to having key-generation as "virtual function" in the various
***_METHOD tables.

1 files changed, 49 insertions, 31 deletions

diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index 918b9237c6..a9ec01d916 100644
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -115,6 +115,11 @@
 #include "bn_lcl.h"
 #include <openssl/rand.h>
 
+/* NB: these functions have been "upgraded", the deprecated versions (which are
+ * compatibility wrappers using these functions) are in bn_depr.c.
+ * - Geoff
+ */
+
 /* The quick sieve algorithm approach to weeding out primes is
  * Philip Zimmermann's, as implemented in PGP.  I have had a read of
  * his comments and implemented my own version.
@@ -129,11 +134,29 @@ static int probable_prime_dh(BIGNUM *rnd, int bits,
 static int probable_prime_dh_safe(BIGNUM *rnd, int bits,
 	const BIGNUM *add, const BIGNUM *rem, BN_CTX *ctx);
 
-BIGNUM *BN_generate_prime(BIGNUM *ret, int bits, int safe,
-	const BIGNUM *add, const BIGNUM *rem,
-	void (*callback)(int,int,void *), void *cb_arg)
+int BN_GENCB_call(BN_GENCB *cb, int a, int b)
+	{
+	/* No callback means continue */
+	if(!cb) return 1;
+	switch(cb->ver)
+		{
+	case 1:
+		/* Deprecated-style callbacks */
+		cb->cb_1(a, b, cb->arg);
+		return 1;
+	case 2:
+		/* New-style callbacks */
+		return cb->cb_2(a, b, cb);
+	default:
+		break;
+		}
+	/* Unrecognised callback type */
+	return 0;
+	}
+
+int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
+	const BIGNUM *add, const BIGNUM *rem, BN_GENCB *cb)
 	{
-	BIGNUM *rnd=NULL;
 	BIGNUM t;
 	int found=0;
 	int i,j,c1=0;
@@ -142,38 +165,34 @@ BIGNUM *BN_generate_prime(BIGNUM *ret, int bits, int safe,
 
 	ctx=BN_CTX_new();
 	if (ctx == NULL) goto err;
-	if (ret == NULL)
-		{
-		if ((rnd=BN_new()) == NULL) goto err;
-		}
-	else
-		rnd=ret;
 	BN_init(&t);
 loop: 
 	/* make a random number and set the top and bottom bits */
 	if (add == NULL)
 		{
-		if (!probable_prime(rnd,bits)) goto err;
+		if (!probable_prime(ret,bits)) goto err;
 		}
 	else
 		{
 		if (safe)
 			{
-			if (!probable_prime_dh_safe(rnd,bits,add,rem,ctx))
+			if (!probable_prime_dh_safe(ret,bits,add,rem,ctx))
 				 goto err;
 			}
 		else
 			{
-			if (!probable_prime_dh(rnd,bits,add,rem,ctx))
+			if (!probable_prime_dh(ret,bits,add,rem,ctx))
 				goto err;
 			}
 		}
-	/* if (BN_mod_word(rnd,(BN_ULONG)3) == 1) goto loop; */
-	if (callback != NULL) callback(0,c1++,cb_arg);
+	/* if (BN_mod_word(ret,(BN_ULONG)3) == 1) goto loop; */
+	if(!BN_GENCB_call(cb, 0, c1++))
+		/* aborted */
+		goto err;
 
 	if (!safe)
 		{
-		i=BN_is_prime_fasttest(rnd,checks,callback,ctx,cb_arg,0);
+		i=BN_is_prime_fasttest_ex(ret,checks,ctx,0,cb);
 		if (i == -1) goto err;
 		if (i == 0) goto loop;
 		}
@@ -183,41 +202,38 @@ loop:
 		 * check that (p-1)/2 is prime.
 		 * Since a prime is odd, We just
 		 * need to divide by 2 */
-		if (!BN_rshift1(&t,rnd)) goto err;
+		if (!BN_rshift1(&t,ret)) goto err;
 
 		for (i=0; i<checks; i++)
 			{
-			j=BN_is_prime_fasttest(rnd,1,callback,ctx,cb_arg,0);
+			j=BN_is_prime_fasttest_ex(ret,1,ctx,0,cb);
 			if (j == -1) goto err;
 			if (j == 0) goto loop;
 
-			j=BN_is_prime_fasttest(&t,1,callback,ctx,cb_arg,0);
+			j=BN_is_prime_fasttest_ex(&t,1,ctx,0,cb);
 			if (j == -1) goto err;
 			if (j == 0) goto loop;
 
-			if (callback != NULL) callback(2,c1-1,cb_arg);
+			if(!BN_GENCB_call(cb, 2, c1-1))
+				goto err;
 			/* We have a safe prime test pass */
 			}
 		}
 	/* we have a prime :-) */
 	found = 1;
 err:
-	if (!found && (ret == NULL) && (rnd != NULL)) BN_free(rnd);
 	BN_free(&t);
 	if (ctx != NULL) BN_CTX_free(ctx);
-	return(found ? rnd : NULL);
+	return found;
 	}
 
-int BN_is_prime(const BIGNUM *a, int checks, void (*callback)(int,int,void *),
-	BN_CTX *ctx_passed, void *cb_arg)
+int BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed, BN_GENCB *cb)
 	{
-	return BN_is_prime_fasttest(a, checks, callback, ctx_passed, cb_arg, 0);
+	return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);
 	}
 
-int BN_is_prime_fasttest(const BIGNUM *a, int checks,
-		void (*callback)(int,int,void *),
-		BN_CTX *ctx_passed, void *cb_arg,
-		int do_trial_division)
+int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
+		int do_trial_division, BN_GENCB *cb)
 	{
 	int i, j, ret = -1;
 	int k;
@@ -240,7 +256,8 @@ int BN_is_prime_fasttest(const BIGNUM *a, int checks,
 		for (i = 1; i < NUMPRIMES; i++)
 			if (BN_mod_word(a, primes[i]) == 0) 
 				return 0;
-		if (callback != NULL) callback(1, -1, cb_arg);
+		if(!BN_GENCB_call(cb, 1, -1))
+			goto err;
 		}
 
 	if (ctx_passed != NULL)
@@ -306,7 +323,8 @@ int BN_is_prime_fasttest(const BIGNUM *a, int checks,
 			ret=0;
 			goto err;
 			}
-		if (callback != NULL) callback(1,i,cb_arg);
+		if(!BN_GENCB_call(cb, 1, i))
+			goto err;
 		}
 	ret=1;
 err: