This commits changes to various parts of libcrypto required by the recent

ENGINE surgery. DH, DSA, RAND, and RSA now use *both* "method" and ENGINE
pointers to manage their hooking with ENGINE. Previously their use of
"method" pointers was replaced by use of ENGINE references. See
crypto/engine/README for details.

Also, remove the ENGINE iterations from evp_test - even when the
cipher/digest code is committed in, this functionality would require a
different set of API calls.

4 files changed, 69 insertions, 127 deletions

diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index 993b539b7a..459dba08b1 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -66,6 +66,7 @@
 #endif
 #include <openssl/bn.h>
 #include <openssl/crypto.h>
+#include <openssl/types.h>
 
 #ifdef OPENSSL_NO_RSA
 #error RSA is disabled.
@@ -122,11 +123,9 @@ struct rsa_st
 	 * this is passed instead of aEVP_PKEY, it is set to 0 */
 	int pad;
 	long version;
-#if 0
-	RSA_METHOD *meth;
-#else
-	struct engine_st *engine;
-#endif
+	const RSA_METHOD *meth;
+	/* functional reference if 'meth' is ENGINE-provided */
+	ENGINE *engine;
 	BIGNUM *n;
 	BIGNUM *e;
 	BIGNUM *d;
@@ -180,11 +179,7 @@ struct rsa_st
 #define RSA_get_app_data(s)             RSA_get_ex_data(s,0)
 
 RSA *	RSA_new(void);
-#if 0
-RSA *	RSA_new_method(RSA_METHOD *method);
-#else
-RSA *	RSA_new_method(struct engine_st *engine);
-#endif
+RSA *	RSA_new_method(ENGINE *engine);
 int	RSA_size(const RSA *);
 RSA *	RSA_generate_key(int bits, unsigned long e,void
 		(*callback)(int,int,void *),void *cb_arg);
@@ -204,14 +199,10 @@ int	RSA_up_ref(RSA *r);
 
 int	RSA_flags(const RSA *r);
 
-void RSA_set_default_openssl_method(const RSA_METHOD *meth);
-const RSA_METHOD *RSA_get_default_openssl_method(void);
+void RSA_set_default_method(const RSA_METHOD *meth);
+const RSA_METHOD *RSA_get_default_method(void);
 const RSA_METHOD *RSA_get_method(const RSA *rsa);
-#if 0
-RSA_METHOD *RSA_set_method(RSA *rsa, RSA_METHOD *meth);
-#else
-int RSA_set_method(RSA *rsa, struct engine_st *engine);
-#endif
+int RSA_set_method(RSA *rsa, const RSA_METHOD *meth);
 
 /* This function needs the memory locking malloc callbacks to be installed */
 int RSA_memory_lock(RSA *r);
diff --git a/crypto/rsa/rsa_eay.c b/crypto/rsa/rsa_eay.c
index 83891df316..d82dd15493 100644
--- a/crypto/rsa/rsa_eay.c
+++ b/crypto/rsa/rsa_eay.c
@@ -100,13 +100,11 @@ const RSA_METHOD *RSA_PKCS1_SSLeay(void)
 static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
-	const RSA_METHOD *meth;
 	BIGNUM f,ret;
 	int i,j,k,num=0,r= -1;
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
-	meth = ENGINE_get_RSA(rsa->engine);
 	BN_init(&f);
 	BN_init(&ret);
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
@@ -172,7 +170,7 @@ static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 			BN_MONT_CTX_free(bn_mont_ctx);
 		}
 		
-	if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
+	if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
 		rsa->_method_mod_n)) goto err;
 
 	/* put in leading 0 bytes if the number is less than the
@@ -199,13 +197,11 @@ err:
 static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
-	const RSA_METHOD *meth;
 	BIGNUM f,ret;
 	int i,j,k,num=0,r= -1;
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
-	meth = ENGINE_get_RSA(rsa->engine);
 	BN_init(&f);
 	BN_init(&ret);
 
@@ -252,10 +248,10 @@ static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
 		(rsa->dmp1 != NULL) &&
 		(rsa->dmq1 != NULL) &&
 		(rsa->iqmp != NULL)) )
-		{ if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
+		{ if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
 	else
 		{
-		if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
+		if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
 		}
 
 	if (rsa->flags & RSA_FLAG_BLINDING)
@@ -284,14 +280,12 @@ err:
 static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
-	const RSA_METHOD *meth;
 	BIGNUM f,ret;
 	int j,num=0,r= -1;
 	unsigned char *p;
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
-	meth = ENGINE_get_RSA(rsa->engine);
 	BN_init(&f);
 	BN_init(&ret);
 	ctx=BN_CTX_new();
@@ -334,10 +328,10 @@ static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
 		(rsa->dmp1 != NULL) &&
 		(rsa->dmq1 != NULL) &&
 		(rsa->iqmp != NULL)) )
-		{ if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
+		{ if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
 	else
 		{
-		if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
+		if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
 			goto err;
 		}
 
@@ -386,14 +380,12 @@ err:
 static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
-	const RSA_METHOD *meth;
 	BIGNUM f,ret;
 	int i,num=0,r= -1;
 	unsigned char *p;
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
-	meth = ENGINE_get_RSA(rsa->engine);
 	BN_init(&f);
 	BN_init(&ret);
 	ctx=BN_CTX_new();
@@ -448,7 +440,7 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 			BN_MONT_CTX_free(bn_mont_ctx);
 		}
 		
-	if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
+	if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
 		rsa->_method_mod_n)) goto err;
 
 	p=buf;
@@ -483,12 +475,10 @@ err:
 
 static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
 	{
-	const RSA_METHOD *meth;
 	BIGNUM r1,m1,vrfy;
 	int ret=0;
 	BN_CTX *ctx;
 
-	meth = ENGINE_get_RSA(rsa->engine);
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
 	BN_init(&m1);
 	BN_init(&r1);
@@ -546,11 +536,11 @@ static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
 		}
 		
 	if (!BN_mod(&r1,I,rsa->q,ctx)) goto err;
-	if (!meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
+	if (!rsa->meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
 		rsa->_method_mod_q)) goto err;
 
 	if (!BN_mod(&r1,I,rsa->p,ctx)) goto err;
-	if (!meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
+	if (!rsa->meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
 		rsa->_method_mod_p)) goto err;
 
 	if (!BN_sub(r0,r0,&m1)) goto err;
@@ -575,7 +565,7 @@ static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
 
 	if (rsa->e && rsa->n)
 		{
-		if (!meth->bn_mod_exp(&vrfy,r0,rsa->e,rsa->n,ctx,NULL)) goto err;
+		if (!rsa->meth->bn_mod_exp(&vrfy,r0,rsa->e,rsa->n,ctx,NULL)) goto err;
 		/* If 'I' was greater than (or equal to) rsa->n, the operation
 		 * will be equivalent to using 'I mod n'. However, the result of
 		 * the verify will *always* be less than 'n' so we don't check
@@ -588,7 +578,7 @@ static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
 			/* 'I' and 'vrfy' aren't congruent mod n. Don't leak
 			 * miscalculated CRT output, just do a raw (slower)
 			 * mod_exp and return that instead. */
-			if (!meth->bn_mod_exp(r0,I,rsa->d,rsa->n,ctx,NULL)) goto err;
+			if (!rsa->meth->bn_mod_exp(r0,I,rsa->d,rsa->n,ctx,NULL)) goto err;
 		}
 	ret=1;
 err:
diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index ef9f924ab4..b81fb6ddb2 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -73,27 +73,13 @@ RSA *RSA_new(void)
 	return(RSA_new_method(NULL));
 	}
 
-void RSA_set_default_openssl_method(const RSA_METHOD *meth)
+void RSA_set_default_method(const RSA_METHOD *meth)
 	{
-	ENGINE *e;
-	/* We'll need to notify the "openssl" ENGINE of this
-	 * change too. We won't bother locking things down at
-	 * our end as there was never any locking in these
-	 * functions! */
-	if(default_RSA_meth != meth)
-		{
-		default_RSA_meth = meth;
-		e = ENGINE_by_id("openssl");
-		if(e)
-			{
-			ENGINE_set_RSA(e, meth);
-			ENGINE_free(e);
-			}
-		}
+	default_RSA_meth = meth;
 	}
 
-const RSA_METHOD *RSA_get_default_openssl_method(void)
-{
+const RSA_METHOD *RSA_get_default_method(void)
+	{
 	if (default_RSA_meth == NULL)
 		{
 #ifdef RSA_NULL
@@ -108,49 +94,32 @@ const RSA_METHOD *RSA_get_default_openssl_method(void)
 		}
 
 	return default_RSA_meth;
-}
+	}
 
 const RSA_METHOD *RSA_get_method(const RSA *rsa)
-{
-	return ENGINE_get_RSA(rsa->engine);
-}
-
-#if 0
-RSA_METHOD *RSA_set_method(RSA *rsa, RSA_METHOD *meth)
-{
-	RSA_METHOD *mtmp;
+	{
+	return rsa->meth;
+	}
+
+int RSA_set_method(RSA *rsa, const RSA_METHOD *meth)
+	{
+	/* NB: The caller is specifically setting a method, so it's not up to us
+	 * to deal with which ENGINE it comes from. */
+	const RSA_METHOD *mtmp;
 	mtmp = rsa->meth;
 	if (mtmp->finish) mtmp->finish(rsa);
+	if (rsa->engine)
+		{
+		ENGINE_finish(rsa->engine);
+		rsa->engine = NULL;
+		}
 	rsa->meth = meth;
 	if (meth->init) meth->init(rsa);
-	return mtmp;
-}
-#else
-int RSA_set_method(RSA *rsa, ENGINE *engine)
-{
-	ENGINE *mtmp;
-	const RSA_METHOD *meth;
-	mtmp = rsa->engine;
-	meth = ENGINE_get_RSA(mtmp);
-	if (!ENGINE_init(engine))
-		return 0;
-	if (meth->finish) meth->finish(rsa);
-	rsa->engine = engine;
-	meth = ENGINE_get_RSA(engine);
-	if (meth->init) meth->init(rsa);
-	/* SHOULD ERROR CHECK THIS!!! */
-	ENGINE_finish(mtmp);
 	return 1;
-}
-#endif
+	}
 
-#if 0
-RSA *RSA_new_method(RSA_METHOD *meth)
-#else
 RSA *RSA_new_method(ENGINE *engine)
-#endif
 	{
-	const RSA_METHOD *meth;
 	RSA *ret;
 
 	ret=(RSA *)OPENSSL_malloc(sizeof(RSA));
@@ -160,25 +129,23 @@ RSA *RSA_new_method(ENGINE *engine)
 		return(NULL);
 		}
 
-	if (engine)
+	ret->meth = RSA_get_default_method();
+	ret->engine = engine;
+	if(!ret->engine)
+		ret->engine = ENGINE_get_default_RSA();
+	if(ret->engine)
 		{
-		if(ENGINE_init(engine))
-			ret->engine = engine;
-		else 
-			ret->engine = NULL;
-		}
-	else
-		ret->engine=ENGINE_get_default_RSA();
-
-	if(ret->engine == NULL)
-		{
-		RSAerr(RSA_F_RSA_NEW_METHOD,ERR_LIB_ENGINE);
-		OPENSSL_free(ret);
-		return NULL;
+		ret->meth = ENGINE_get_RSA(ret->engine);
+		if(!ret->meth)
+			{
+			RSAerr(RSA_F_RSA_NEW_METHOD,
+				ERR_R_ENGINE_LIB);
+			ENGINE_finish(ret->engine);
+			OPENSSL_free(ret);
+			return NULL;
+			}
 		}
 
-	meth = ENGINE_get_RSA(ret->engine);
-
 	ret->pad=0;
 	ret->version=0;
 	ret->n=NULL;
@@ -195,9 +162,9 @@ RSA *RSA_new_method(ENGINE *engine)
 	ret->_method_mod_q=NULL;
 	ret->blinding=NULL;
 	ret->bignum_data=NULL;
-	ret->flags=meth->flags;
+	ret->flags=ret->meth->flags;
 	CRYPTO_new_ex_data(CRYPTO_EX_INDEX_RSA, ret, &ret->ex_data);
-	if ((meth->init != NULL) && !meth->init(ret))
+	if ((ret->meth->init != NULL) && !ret->meth->init(ret))
 		{
 		CRYPTO_free_ex_data(CRYPTO_EX_INDEX_RSA, ret, &ret->ex_data);
 		OPENSSL_free(ret);
@@ -208,7 +175,6 @@ RSA *RSA_new_method(ENGINE *engine)
 
 void RSA_free(RSA *r)
 	{
-	const RSA_METHOD *meth;
 	int i;
 
 	if (r == NULL) return;
@@ -226,10 +192,10 @@ void RSA_free(RSA *r)
 		}
 #endif
 
-	meth = ENGINE_get_RSA(r->engine);
-	if (meth->finish != NULL)
-		meth->finish(r);
-	ENGINE_finish(r->engine);
+	if (r->meth->finish)
+		r->meth->finish(r);
+	if (r->engine)
+		ENGINE_finish(r->engine);
 
 	CRYPTO_free_ex_data(CRYPTO_EX_INDEX_RSA, r, &r->ex_data);
 
@@ -287,34 +253,30 @@ int RSA_size(const RSA *r)
 int RSA_public_encrypt(int flen, const unsigned char *from, unsigned char *to,
 	     RSA *rsa, int padding)
 	{
-	return(ENGINE_get_RSA(rsa->engine)->rsa_pub_enc(flen,
-		from, to, rsa, padding));
+	return(rsa->meth->rsa_pub_enc(flen, from, to, rsa, padding));
 	}
 
 int RSA_private_encrypt(int flen, const unsigned char *from, unsigned char *to,
 	     RSA *rsa, int padding)
 	{
-	return(ENGINE_get_RSA(rsa->engine)->rsa_priv_enc(flen,
-		from, to, rsa, padding));
+	return(rsa->meth->rsa_priv_enc(flen, from, to, rsa, padding));
 	}
 
 int RSA_private_decrypt(int flen, const unsigned char *from, unsigned char *to,
 	     RSA *rsa, int padding)
 	{
-	return(ENGINE_get_RSA(rsa->engine)->rsa_priv_dec(flen,
-		from, to, rsa, padding));
+	return(rsa->meth->rsa_priv_dec(flen, from, to, rsa, padding));
 	}
 
 int RSA_public_decrypt(int flen, const unsigned char *from, unsigned char *to,
 	     RSA *rsa, int padding)
 	{
-	return(ENGINE_get_RSA(rsa->engine)->rsa_pub_dec(flen,
-		from, to, rsa, padding));
+	return(rsa->meth->rsa_pub_dec(flen, from, to, rsa, padding));
 	}
 
 int RSA_flags(const RSA *r)
 	{
-	return((r == NULL)?0:ENGINE_get_RSA(r->engine)->flags);
+	return((r == NULL)?0:r->meth->flags);
 	}
 
 void RSA_blinding_off(RSA *rsa)
@@ -348,8 +310,7 @@ int RSA_blinding_on(RSA *rsa, BN_CTX *p_ctx)
 	if (!BN_rand_range(A,rsa->n)) goto err;
 	if ((Ai=BN_mod_inverse(NULL,A,rsa->n,ctx)) == NULL) goto err;
 
-	if (!ENGINE_get_RSA(rsa->engine)->bn_mod_exp(A,A,
-		rsa->e,rsa->n,ctx,rsa->_method_mod_n))
+	if (!rsa->meth->bn_mod_exp(A,A,rsa->e,rsa->n,ctx,rsa->_method_mod_n))
 	    goto err;
 	rsa->blinding=BN_BLINDING_new(A,Ai,rsa->n);
 	rsa->flags|=RSA_FLAG_BLINDING;
@@ -405,4 +366,3 @@ int RSA_memory_lock(RSA *r)
 	r->bignum_data=p;
 	return(1);
 	}
-
diff --git a/crypto/rsa/rsa_test.c b/crypto/rsa/rsa_test.c
index 5dee24b001..b8b462d33b 100644
--- a/crypto/rsa/rsa_test.c
+++ b/crypto/rsa/rsa_test.c
@@ -16,6 +16,7 @@ int main(int argc, char *argv[])
 }
 #else
 #include <openssl/rsa.h>
+#include <openssl/engine.h>
 
 #define SetKey \
   key->n = BN_bin2bn(n, sizeof(n)-1, key->n); \
@@ -219,12 +220,12 @@ int main(int argc, char *argv[])
     int clen = 0;
     int num;
 
-    RAND_seed(rnd_seed, sizeof rnd_seed); /* or OAEP may fail */
-
     CRYPTO_malloc_debug_init();
     CRYPTO_dbg_set_options(V_CRYPTO_MDEBUG_ALL);
     CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-	
+
+    RAND_seed(rnd_seed, sizeof rnd_seed); /* or OAEP may fail */
+
     plen = sizeof(ptext_ex) - 1;
 
     for (v = 0; v < 3; v++)
@@ -310,7 +311,7 @@ int main(int argc, char *argv[])
     CRYPTO_cleanup_all_ex_data();
     ERR_remove_state(0);
 
-    CRYPTO_mem_leaks_fp(stdout);
+    CRYPTO_mem_leaks_fp(stderr);
 
     return err;
     }