This is a first-cut at improving the callback mechanisms used in

key-generation and prime-checking functions. Rather than explicitly passing
callback functions and caller-defined context data for the callbacks, a new
structure BN_GENCB is defined that encapsulates this; a pointer to the
structure is passed to all such functions instead.

This wrapper structure allows the encapsulation of "old" and "new" style
callbacks - "new" callbacks return a boolean result on the understanding
that returning FALSE should terminate keygen/primality processing.  The
BN_GENCB abstraction will allow future callback modifications without
needing to break binary compatibility nor change the API function
prototypes. The new API functions have been given names ending in "_ex" and
the old functions are implemented as wrappers to the new ones.  The
OPENSSL_NO_DEPRECATED symbol has been introduced so that, if defined,
declaration of the older functions will be skipped. NB: Some
openssl-internal code will stick with the older callbacks for now, so
appropriate "#undef" logic will be put in place - this is in case the user
is *building* openssl (rather than *including* its headers) with this
symbol defined.

There is another change in the new _ex functions; the key-generation
functions do not return key structures but operate on structures passed by
the caller, the return value is a boolean. This will allow for a smoother
transition to having key-generation as "virtual function" in the various
***_METHOD tables.

5 files changed, 125 insertions, 32 deletions

diff --git a/crypto/rsa/Makefile.ssl b/crypto/rsa/Makefile.ssl
index 7e4c6db986..eeb0c9115c 100644
--- a/crypto/rsa/Makefile.ssl
+++ b/crypto/rsa/Makefile.ssl
@@ -25,10 +25,10 @@ APPS=
 LIB=$(TOP)/libcrypto.a
 LIBSRC= rsa_eay.c rsa_gen.c rsa_lib.c rsa_sign.c rsa_saos.c rsa_err.c \
 	rsa_pk1.c rsa_ssl.c rsa_none.c rsa_oaep.c rsa_chk.c rsa_null.c \
-	rsa_asn1.c
+	rsa_asn1.c rsa_depr.c
 LIBOBJ= rsa_eay.o rsa_gen.o rsa_lib.o rsa_sign.o rsa_saos.o rsa_err.o \
 	rsa_pk1.o rsa_ssl.o rsa_none.o rsa_oaep.o rsa_chk.o rsa_null.o \
-	rsa_asn1.o
+	rsa_asn1.o rsa_depr.o
 
 SRC= $(LIBSRC)
 
diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index b2e25e4e7c..cdf514c009 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -183,8 +183,16 @@ struct rsa_st
 RSA *	RSA_new(void);
 RSA *	RSA_new_method(ENGINE *engine);
 int	RSA_size(const RSA *);
+
+/* Deprecated version */
+#ifndef OPENSSL_NO_DEPRECATED
 RSA *	RSA_generate_key(int bits, unsigned long e,void
 		(*callback)(int,int,void *),void *cb_arg);
+#endif /* !defined(OPENSSL_NO_DEPRECATED) */
+
+/* New version */
+int	RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e, BN_GENCB *cb);
+
 int	RSA_check_key(const RSA *);
 	/* next 4 return -1 on error */
 int	RSA_public_encrypt(int flen, const unsigned char *from,
diff --git a/crypto/rsa/rsa_chk.c b/crypto/rsa/rsa_chk.c
index 002f2cb487..9d848db8c6 100644
--- a/crypto/rsa/rsa_chk.c
+++ b/crypto/rsa/rsa_chk.c
@@ -75,7 +75,7 @@ int RSA_check_key(const RSA *key)
 		}
 	
 	/* p prime? */
-	r = BN_is_prime(key->p, BN_prime_checks, NULL, NULL, NULL);
+	r = BN_is_prime_ex(key->p, BN_prime_checks, NULL, NULL);
 	if (r != 1)
 		{
 		ret = r;
@@ -85,7 +85,7 @@ int RSA_check_key(const RSA *key)
 		}
 	
 	/* q prime? */
-	r = BN_is_prime(key->q, BN_prime_checks, NULL, NULL, NULL);
+	r = BN_is_prime_ex(key->q, BN_prime_checks, NULL, NULL);
 	if (r != 1)
 		{
 		ret = r;
diff --git a/crypto/rsa/rsa_depr.c b/crypto/rsa/rsa_depr.c
new file mode 100644
index 0000000000..25fa954393
--- /dev/null
+++ b/crypto/rsa/rsa_depr.c
@@ -0,0 +1,83 @@
+/* crypto/rsa/rsa_depr.c */
+/* ====================================================================
+ * Copyright (c) 1998-2002 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+/* NB: This file contains deprecated functions (compatibility wrappers to the
+ * "new" versions). */
+
+#include <stdio.h>
+#include <time.h>
+#include "cryptlib.h"
+#include <openssl/bn.h>
+#include <openssl/rsa.h>
+
+RSA *RSA_generate_key(int bits, unsigned long e_value,
+	     void (*callback)(int,int,void *), void *cb_arg)
+	{
+	BN_GENCB cb;
+	RSA *rsa;
+
+	if((rsa=RSA_new()) == NULL)
+		return 0;
+
+	cb.ver = 1;
+	cb.arg = cb_arg;
+	cb.cb_1 = callback;
+
+	if(RSA_generate_key_ex(rsa, bits, e_value, &cb))
+		return rsa;
+	RSA_free(rsa);
+	return 0;
+	}
+
diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 00c25adbc5..a45b9aab5c 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -56,16 +56,20 @@
  * [including the GNU Public Licence.]
  */
 
+
+/* NB: these functions have been "upgraded", the deprecated versions (which are
+ * compatibility wrappers using these functions) are in rsa_depr.c.
+ * - Geoff
+ */
+
 #include <stdio.h>
 #include <time.h>
 #include "cryptlib.h"
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-RSA *RSA_generate_key(int bits, unsigned long e_value,
-	     void (*callback)(int,int,void *), void *cb_arg)
+int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
 	{
-	RSA *rsa=NULL;
 	BIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;
 	int bitsp,bitsq,ok= -1,n=0,i;
 	BN_CTX *ctx=NULL,*ctx2=NULL;
@@ -83,12 +87,16 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 
 	bitsp=(bits+1)/2;
 	bitsq=bits-bitsp;
-	rsa=RSA_new();
-	if (rsa == NULL) goto err;
 
-	/* set e */ 
-	rsa->e=BN_new();
-	if (rsa->e == NULL) goto err;
+	/* We need the RSA components non-NULL */
+	if(!rsa->n && ((rsa->n=BN_new()) == NULL)) goto err;
+	if(!rsa->d && ((rsa->d=BN_new()) == NULL)) goto err;
+	if(!rsa->e && ((rsa->e=BN_new()) == NULL)) goto err;
+	if(!rsa->p && ((rsa->p=BN_new()) == NULL)) goto err;
+	if(!rsa->q && ((rsa->q=BN_new()) == NULL)) goto err;
+	if(!rsa->dmp1 && ((rsa->dmp1=BN_new()) == NULL)) goto err;
+	if(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;
+	if(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;
 
 #if 1
 	/* The problem is when building with 8, 16, or 32 BN_ULONG,
@@ -105,27 +113,29 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 	/* generate p and q */
 	for (;;)
 		{
-		rsa->p=BN_generate_prime(NULL,bitsp,0,NULL,NULL,callback,cb_arg);
-		if (rsa->p == NULL) goto err;
+		if(!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
+			goto err;
 		if (!BN_sub(r2,rsa->p,BN_value_one())) goto err;
 		if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
 		if (BN_is_one(r1)) break;
-		if (callback != NULL) callback(2,n++,cb_arg);
-		BN_free(rsa->p);
+		if(!BN_GENCB_call(cb, 2, n++))
+			goto err;
 		}
-	if (callback != NULL) callback(3,0,cb_arg);
+	if(!BN_GENCB_call(cb, 3, 0))
+		goto err;
 	for (;;)
 		{
-		rsa->q=BN_generate_prime(NULL,bitsq,0,NULL,NULL,callback,cb_arg);
-		if (rsa->q == NULL) goto err;
+		if(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
+			goto err;
 		if (!BN_sub(r2,rsa->q,BN_value_one())) goto err;
 		if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
 		if (BN_is_one(r1) && (BN_cmp(rsa->p,rsa->q) != 0))
 			break;
-		if (callback != NULL) callback(2,n++,cb_arg);
-		BN_free(rsa->q);
+		if(!BN_GENCB_call(cb, 2, n++))
+			goto err;
 		}
-	if (callback != NULL) callback(3,1,cb_arg);
+	if(!BN_GENCB_call(cb, 3, 1))
+		goto err;
 	if (BN_cmp(rsa->p,rsa->q) < 0)
 		{
 		tmp=rsa->p;
@@ -134,8 +144,6 @@ RSA *RSA_generate_key(int bits, unsigned long e_value,
 		}
 
 	/* calculate n */
-	rsa->n=BN_new();
-	if (rsa->n == NULL) goto err;
 	if (!BN_mul(rsa->n,rsa->p,rsa->q,ctx)) goto err;
 
 	/* calculate d */
@@ -185,13 +193,7 @@ err:
 	BN_CTX_end(ctx);
 	BN_CTX_free(ctx);
 	BN_CTX_free(ctx2);
-	
-	if (!ok)
-		{
-		if (rsa != NULL) RSA_free(rsa);
-		return(NULL);
-		}
-	else
-		return(rsa);
+
+	return ok;
 	}