This is a first-cut at improving the callback mechanisms used in

key-generation and prime-checking functions. Rather than explicitly passing
callback functions and caller-defined context data for the callbacks, a new
structure BN_GENCB is defined that encapsulates this; a pointer to the
structure is passed to all such functions instead.

This wrapper structure allows the encapsulation of "old" and "new" style
callbacks - "new" callbacks return a boolean result on the understanding
that returning FALSE should terminate keygen/primality processing.  The
BN_GENCB abstraction will allow future callback modifications without
needing to break binary compatibility nor change the API function
prototypes. The new API functions have been given names ending in "_ex" and
the old functions are implemented as wrappers to the new ones.  The
OPENSSL_NO_DEPRECATED symbol has been introduced so that, if defined,
declaration of the older functions will be skipped. NB: Some
openssl-internal code will stick with the older callbacks for now, so
appropriate "#undef" logic will be put in place - this is in case the user
is *building* openssl (rather than *including* its headers) with this
symbol defined.

There is another change in the new _ex functions; the key-generation
functions do not return key structures but operate on structures passed by
the caller, the return value is a boolean. This will allow for a smoother
transition to having key-generation as "virtual function" in the various
***_METHOD tables.

1 files changed, 30 insertions, 28 deletions

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 00c25adbc5..a45b9aab5c 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -56,16 +56,20 @@
  * [including the GNU Public Licence.]
  */
 
+
+/* NB: these functions have been "upgraded", the deprecated versions (which are
+ * compatibility wrappers using these functions) are in rsa_depr.c.
+ * - Geoff
+ */
+
 #include <stdio.h>
 #include <time.h>
 #include "cryptlib.h"
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-RSA *RSA_generate_key(int bits, unsigned long e_value,
-	     void (*callback)(int,int,void *), void *cb_arg)
+int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
 	{
-	RSA *rsa=NULL;
 	BIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;
 	int bitsp,bitsq,ok= -1,n=0,i;
 	BN_CTX *ctx=NULL,*ctx2=NULL;
@@ -83,12 +87,16 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 
 	bitsp=(bits+1)/2;
 	bitsq=bits-bitsp;
-	rsa=RSA_new();
-	if (rsa == NULL) goto err;
 
-	/* set e */ 
-	rsa->e=BN_new();
-	if (rsa->e == NULL) goto err;
+	/* We need the RSA components non-NULL */
+	if(!rsa->n && ((rsa->n=BN_new()) == NULL)) goto err;
+	if(!rsa->d && ((rsa->d=BN_new()) == NULL)) goto err;
+	if(!rsa->e && ((rsa->e=BN_new()) == NULL)) goto err;
+	if(!rsa->p && ((rsa->p=BN_new()) == NULL)) goto err;
+	if(!rsa->q && ((rsa->q=BN_new()) == NULL)) goto err;
+	if(!rsa->dmp1 && ((rsa->dmp1=BN_new()) == NULL)) goto err;
+	if(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;
+	if(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;
 
 #if 1
 	/* The problem is when building with 8, 16, or 32 BN_ULONG,
@@ -105,27 +113,29 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 	/* generate p and q */
 	for (;;)
 		{
-		rsa->p=BN_generate_prime(NULL,bitsp,0,NULL,NULL,callback,cb_arg);
-		if (rsa->p == NULL) goto err;
+		if(!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
+			goto err;
 		if (!BN_sub(r2,rsa->p,BN_value_one())) goto err;
 		if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
 		if (BN_is_one(r1)) break;
-		if (callback != NULL) callback(2,n++,cb_arg);
-		BN_free(rsa->p);
+		if(!BN_GENCB_call(cb, 2, n++))
+			goto err;
 		}
-	if (callback != NULL) callback(3,0,cb_arg);
+	if(!BN_GENCB_call(cb, 3, 0))
+		goto err;
 	for (;;)
 		{
-		rsa->q=BN_generate_prime(NULL,bitsq,0,NULL,NULL,callback,cb_arg);
-		if (rsa->q == NULL) goto err;
+		if(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
+			goto err;
 		if (!BN_sub(r2,rsa->q,BN_value_one())) goto err;
 		if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
 		if (BN_is_one(r1) && (BN_cmp(rsa->p,rsa->q) != 0))
 			break;
-		if (callback != NULL) callback(2,n++,cb_arg);
-		BN_free(rsa->q);
+		if(!BN_GENCB_call(cb, 2, n++))
+			goto err;
 		}
-	if (callback != NULL) callback(3,1,cb_arg);
+	if(!BN_GENCB_call(cb, 3, 1))
+		goto err;
 	if (BN_cmp(rsa->p,rsa->q) < 0)
 		{
 		tmp=rsa->p;
@@ -134,8 +144,6 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 		}
 
 	/* calculate n */
-	rsa->n=BN_new();
-	if (rsa->n == NULL) goto err;
 	if (!BN_mul(rsa->n,rsa->p,rsa->q,ctx)) goto err;
 
 	/* calculate d */
@@ -185,13 +193,7 @@ err:
 	BN_CTX_end(ctx);
 	BN_CTX_free(ctx);
 	BN_CTX_free(ctx2);
-	
-	if (!ok)
-		{
-		if (rsa != NULL) RSA_free(rsa);
-		return(NULL);
-		}
-	else
-		return(rsa);
+
+	return ok;
 	}