Update PKCS#7 enveloped data to new API.

3 files changed, 65 insertions, 38 deletions

diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index 67bb8264e0..026dd540d6 100644
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -138,6 +138,66 @@ static int PKCS7_bio_add_digest(BIO **pbio, X509_ALGOR *alg)
 
 	}
 
+static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri,
+					unsigned char *key, int keylen)
+	{
+	EVP_PKEY_CTX *pctx = NULL;
+	EVP_PKEY *pkey = NULL;
+	unsigned char *ek = NULL;
+	int ret = 0;
+	int eklen;
+
+	pkey = X509_get_pubkey(ri->cert);
+
+	if (!pkey)
+		return 0;
+
+	pctx = EVP_PKEY_CTX_new(pkey, NULL);
+	if (!pctx)
+		return 0;
+
+	if (EVP_PKEY_encrypt_init(pctx) <= 0)
+		goto err;
+
+	if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_ENCRYPT,
+				EVP_PKEY_CTRL_PKCS7_ENCRYPT, 0, ri) <= 0)
+		{
+		PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, PKCS7_R_CTRL_ERROR);
+		goto err;
+		}
+
+	if (EVP_PKEY_encrypt(pctx, NULL, &eklen, key, keylen) <= 0)
+		goto err;
+
+	ek = OPENSSL_malloc(eklen);
+
+	if (ek == NULL)
+		{
+		PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, ERR_R_MALLOC_FAILURE);
+		goto err;
+		}
+
+	if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0)
+		goto err;
+
+	ASN1_STRING_set0(ri->enc_key, ek, eklen);
+	ek = NULL;
+
+	ret = 1;
+
+	err:
+	if (pkey)
+		EVP_PKEY_free(pkey);
+	if (pctx)
+		EVP_PKEY_CTX_free(pctx);
+	if (ek)
+		OPENSSL_free(ek);
+	return ret;
+
+	}
+
+
+
 BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 	{
 	int i;
@@ -148,7 +208,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 	STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
 	X509_ALGOR *xalg=NULL;
 	PKCS7_RECIP_INFO *ri=NULL;
-	EVP_PKEY *pkey;
 	ASN1_OCTET_STRING *os=NULL;
 
 	i=OBJ_obj2nid(p7->type);
@@ -204,8 +263,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 		unsigned char key[EVP_MAX_KEY_LENGTH];
 		unsigned char iv[EVP_MAX_IV_LENGTH];
 		int keylen,ivlen;
-		int jj,max;
-		unsigned char *tmp;
 		EVP_CIPHER_CTX *ctx;
 
 		if ((btmp=BIO_new(BIO_f_cipher())) == NULL)
@@ -233,46 +290,12 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 		}
 
 		/* Lets do the pub key stuff :-) */
-		max=0;
-		for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
-			{
-			ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-			if (ri->cert == NULL)
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,PKCS7_R_MISSING_CERIPEND_INFO);
-				goto err;
-				}
-			pkey=X509_get_pubkey(ri->cert);
-			jj=EVP_PKEY_size(pkey);
-			EVP_PKEY_free(pkey);
-			if (max < jj) max=jj;
-			}
-		if ((tmp=(unsigned char *)OPENSSL_malloc(max)) == NULL)
-			{
-			PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_MALLOC_FAILURE);
-			goto err;
-			}
 		for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
 			{
 			ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-			pkey=X509_get_pubkey(ri->cert);
-			jj=EVP_PKEY_encrypt_old(tmp,key,keylen,pkey);
-			EVP_PKEY_free(pkey);
-			if (jj <= 0)
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_EVP_LIB);
-				OPENSSL_free(tmp);
-				goto err;
-				}
-			if (!M_ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj))
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,
-					ERR_R_MALLOC_FAILURE);
-				OPENSSL_free(tmp);
+			if (pkcs7_encode_rinfo(ri, key, keylen) <= 0)
 				goto err;
-				}
 			}
-		OPENSSL_free(tmp);
 		OPENSSL_cleanse(key, keylen);
 
 		if (out == NULL)
diff --git a/crypto/pkcs7/pkcs7.h b/crypto/pkcs7/pkcs7.h
index 180aefb23e..9867753fea 100644
--- a/crypto/pkcs7/pkcs7.h
+++ b/crypto/pkcs7/pkcs7.h
@@ -404,6 +404,7 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_F_PKCS7_DATASIGN				 106
 #define PKCS7_F_PKCS7_DATAVERIFY			 107
 #define PKCS7_F_PKCS7_DECRYPT				 114
+#define PKCS7_F_PKCS7_ENCODE_RINFO			 132
 #define PKCS7_F_PKCS7_ENCRYPT				 115
 #define PKCS7_F_PKCS7_FIND_DIGEST			 127
 #define PKCS7_F_PKCS7_GET0_SIGNERS			 124
@@ -425,6 +426,7 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER		 144
 #define PKCS7_R_CIPHER_NOT_INITIALIZED			 116
 #define PKCS7_R_CONTENT_AND_DATA_PRESENT		 118
+#define PKCS7_R_CTRL_ERROR				 152
 #define PKCS7_R_DECODE_ERROR				 130
 #define PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH		 100
 #define PKCS7_R_DECRYPT_ERROR				 119
diff --git a/crypto/pkcs7/pkcs7err.c b/crypto/pkcs7/pkcs7err.c
index e7950698b6..e6eb206d02 100644
--- a/crypto/pkcs7/pkcs7err.c
+++ b/crypto/pkcs7/pkcs7err.c
@@ -86,6 +86,7 @@ static ERR_STRING_DATA PKCS7_str_functs[]=
 {ERR_FUNC(PKCS7_F_PKCS7_DATASIGN),	"PKCS7_DATASIGN"},
 {ERR_FUNC(PKCS7_F_PKCS7_DATAVERIFY),	"PKCS7_dataVerify"},
 {ERR_FUNC(PKCS7_F_PKCS7_DECRYPT),	"PKCS7_decrypt"},
+{ERR_FUNC(PKCS7_F_PKCS7_ENCODE_RINFO),	"PKCS7_ENCODE_RINFO"},
 {ERR_FUNC(PKCS7_F_PKCS7_ENCRYPT),	"PKCS7_encrypt"},
 {ERR_FUNC(PKCS7_F_PKCS7_FIND_DIGEST),	"PKCS7_FIND_DIGEST"},
 {ERR_FUNC(PKCS7_F_PKCS7_GET0_SIGNERS),	"PKCS7_GET0_SIGNERS"},
@@ -110,6 +111,7 @@ static ERR_STRING_DATA PKCS7_str_reasons[]=
 {ERR_REASON(PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER),"cipher has no object identifier"},
 {ERR_REASON(PKCS7_R_CIPHER_NOT_INITIALIZED),"cipher not initialized"},
 {ERR_REASON(PKCS7_R_CONTENT_AND_DATA_PRESENT),"content and data present"},
+{ERR_REASON(PKCS7_R_CTRL_ERROR)          ,"ctrl error"},
 {ERR_REASON(PKCS7_R_DECODE_ERROR)        ,"decode error"},
 {ERR_REASON(PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH),"decrypted key is wrong length"},
 {ERR_REASON(PKCS7_R_DECRYPT_ERROR)       ,"decrypt error"},