Update PKCS#7 enveloped data to new API.

8 files changed, 90 insertions, 40 deletions

diff --git a/CHANGES b/CHANGES
index 991e4b0972..f1f3894389 100644
--- a/CHANGES
+++ b/CHANGES
@@ -2,7 +2,14 @@
  OpenSSL CHANGES
  _______________
 
- Changes between 0.9.8a and 0.9.9  [xx XXX xxxx]
+ Changes between 0.9.8b and 0.9.9  [xx XXX xxxx]
+
+  *) Update PKCS#7 enveloped data routines to use new API. This is now
+     supported by any public key method supporting the encrypt operation. A
+     ctrl is added to allow the public key algorithm to examine or modify
+     the PKCS#7 RecipientInfo structure if it needs to: for RSA this is
+     a no op.
+     [Steve Henson]
 
   *) Add a ctrl to asn1 method to allow a public key algorithm to express
      a default digest type to use. In most cases this will be SHA1 but some
diff --git a/crypto/asn1/asn1.h b/crypto/asn1/asn1.h
index 912619efc7..02d7829a16 100644
--- a/crypto/asn1/asn1.h
+++ b/crypto/asn1/asn1.h
@@ -765,6 +765,7 @@ int 		ASN1_STRING_cmp(const ASN1_STRING *a, const ASN1_STRING *b);
   /* Since this is used to store all sorts of things, via macros, for now, make
      its data void * */
 int 		ASN1_STRING_set(ASN1_STRING *str, const void *data, int len);
+void		ASN1_STRING_set0(ASN1_STRING *str, void *data, int len);
 int ASN1_STRING_length(const ASN1_STRING *x);
 void ASN1_STRING_length_set(ASN1_STRING *x, int n);
 int ASN1_STRING_type(ASN1_STRING *x);
diff --git a/crypto/asn1/asn1_lib.c b/crypto/asn1/asn1_lib.c
index 69ad869c97..41e1126e77 100644
--- a/crypto/asn1/asn1_lib.c
+++ b/crypto/asn1/asn1_lib.c
@@ -393,6 +393,14 @@ int ASN1_STRING_set(ASN1_STRING *str, const void *_data, int len)
 	return(1);
 	}
 
+void ASN1_STRING_set0(ASN1_STRING *str, void *data, int len)
+	{
+	if (str->data)
+		OPENSSL_free(str->data);
+	str->data = data;
+	str->length = len;
+	}
+
 ASN1_STRING *ASN1_STRING_new(void)
 	{
 	return(ASN1_STRING_type_new(V_ASN1_OCTET_STRING));
diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h
index ff95a34b49..ae7bee8097 100644
--- a/crypto/evp/evp.h
+++ b/crypto/evp/evp.h
@@ -882,7 +882,10 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
 					EVP_PKEY_CTRL_MD, 0, (void *)md)
 
 #define EVP_PKEY_CTRL_MD		1
-#define EVP_PKEY_CTRL_PEER_KEY		2
+#define EVP_PKEY_CTRL_PEER_KEY		2	
+
+#define EVP_PKEY_CTRL_PKCS7_ENCRYPT	3	
+#define EVP_PKEY_CTRL_PKCS7_DECRYPT	4	
 
 #define EVP_PKEY_ALG_CTRL		0x1000
 
diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index 67bb8264e0..026dd540d6 100644
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -138,6 +138,66 @@ static int PKCS7_bio_add_digest(BIO **pbio, X509_ALGOR *alg)
 
 	}
 
+static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri,
+					unsigned char *key, int keylen)
+	{
+	EVP_PKEY_CTX *pctx = NULL;
+	EVP_PKEY *pkey = NULL;
+	unsigned char *ek = NULL;
+	int ret = 0;
+	int eklen;
+
+	pkey = X509_get_pubkey(ri->cert);
+
+	if (!pkey)
+		return 0;
+
+	pctx = EVP_PKEY_CTX_new(pkey, NULL);
+	if (!pctx)
+		return 0;
+
+	if (EVP_PKEY_encrypt_init(pctx) <= 0)
+		goto err;
+
+	if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_ENCRYPT,
+				EVP_PKEY_CTRL_PKCS7_ENCRYPT, 0, ri) <= 0)
+		{
+		PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, PKCS7_R_CTRL_ERROR);
+		goto err;
+		}
+
+	if (EVP_PKEY_encrypt(pctx, NULL, &eklen, key, keylen) <= 0)
+		goto err;
+
+	ek = OPENSSL_malloc(eklen);
+
+	if (ek == NULL)
+		{
+		PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, ERR_R_MALLOC_FAILURE);
+		goto err;
+		}
+
+	if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0)
+		goto err;
+
+	ASN1_STRING_set0(ri->enc_key, ek, eklen);
+	ek = NULL;
+
+	ret = 1;
+
+	err:
+	if (pkey)
+		EVP_PKEY_free(pkey);
+	if (pctx)
+		EVP_PKEY_CTX_free(pctx);
+	if (ek)
+		OPENSSL_free(ek);
+	return ret;
+
+	}
+
+
+
 BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 	{
 	int i;
@@ -148,7 +208,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 	STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
 	X509_ALGOR *xalg=NULL;
 	PKCS7_RECIP_INFO *ri=NULL;
-	EVP_PKEY *pkey;
 	ASN1_OCTET_STRING *os=NULL;
 
 	i=OBJ_obj2nid(p7->type);
@@ -204,8 +263,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 		unsigned char key[EVP_MAX_KEY_LENGTH];
 		unsigned char iv[EVP_MAX_IV_LENGTH];
 		int keylen,ivlen;
-		int jj,max;
-		unsigned char *tmp;
 		EVP_CIPHER_CTX *ctx;
 
 		if ((btmp=BIO_new(BIO_f_cipher())) == NULL)
@@ -233,46 +290,12 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 		}
 
 		/* Lets do the pub key stuff :-) */
-		max=0;
-		for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
-			{
-			ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-			if (ri->cert == NULL)
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,PKCS7_R_MISSING_CERIPEND_INFO);
-				goto err;
-				}
-			pkey=X509_get_pubkey(ri->cert);
-			jj=EVP_PKEY_size(pkey);
-			EVP_PKEY_free(pkey);
-			if (max < jj) max=jj;
-			}
-		if ((tmp=(unsigned char *)OPENSSL_malloc(max)) == NULL)
-			{
-			PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_MALLOC_FAILURE);
-			goto err;
-			}
 		for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
 			{
 			ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-			pkey=X509_get_pubkey(ri->cert);
-			jj=EVP_PKEY_encrypt_old(tmp,key,keylen,pkey);
-			EVP_PKEY_free(pkey);
-			if (jj <= 0)
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_EVP_LIB);
-				OPENSSL_free(tmp);
-				goto err;
-				}
-			if (!M_ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj))
-				{
-				PKCS7err(PKCS7_F_PKCS7_DATAINIT,
-					ERR_R_MALLOC_FAILURE);
-				OPENSSL_free(tmp);
+			if (pkcs7_encode_rinfo(ri, key, keylen) <= 0)
 				goto err;
-				}
 			}
-		OPENSSL_free(tmp);
 		OPENSSL_cleanse(key, keylen);
 
 		if (out == NULL)
diff --git a/crypto/pkcs7/pkcs7.h b/crypto/pkcs7/pkcs7.h
index 180aefb23e..9867753fea 100644
--- a/crypto/pkcs7/pkcs7.h
+++ b/crypto/pkcs7/pkcs7.h
@@ -404,6 +404,7 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_F_PKCS7_DATASIGN				 106
 #define PKCS7_F_PKCS7_DATAVERIFY			 107
 #define PKCS7_F_PKCS7_DECRYPT				 114
+#define PKCS7_F_PKCS7_ENCODE_RINFO			 132
 #define PKCS7_F_PKCS7_ENCRYPT				 115
 #define PKCS7_F_PKCS7_FIND_DIGEST			 127
 #define PKCS7_F_PKCS7_GET0_SIGNERS			 124
@@ -425,6 +426,7 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER		 144
 #define PKCS7_R_CIPHER_NOT_INITIALIZED			 116
 #define PKCS7_R_CONTENT_AND_DATA_PRESENT		 118
+#define PKCS7_R_CTRL_ERROR				 152
 #define PKCS7_R_DECODE_ERROR				 130
 #define PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH		 100
 #define PKCS7_R_DECRYPT_ERROR				 119
diff --git a/crypto/pkcs7/pkcs7err.c b/crypto/pkcs7/pkcs7err.c
index e7950698b6..e6eb206d02 100644
--- a/crypto/pkcs7/pkcs7err.c
+++ b/crypto/pkcs7/pkcs7err.c
@@ -86,6 +86,7 @@ static ERR_STRING_DATA PKCS7_str_functs[]=
 {ERR_FUNC(PKCS7_F_PKCS7_DATASIGN),	"PKCS7_DATASIGN"},
 {ERR_FUNC(PKCS7_F_PKCS7_DATAVERIFY),	"PKCS7_dataVerify"},
 {ERR_FUNC(PKCS7_F_PKCS7_DECRYPT),	"PKCS7_decrypt"},
+{ERR_FUNC(PKCS7_F_PKCS7_ENCODE_RINFO),	"PKCS7_ENCODE_RINFO"},
 {ERR_FUNC(PKCS7_F_PKCS7_ENCRYPT),	"PKCS7_encrypt"},
 {ERR_FUNC(PKCS7_F_PKCS7_FIND_DIGEST),	"PKCS7_FIND_DIGEST"},
 {ERR_FUNC(PKCS7_F_PKCS7_GET0_SIGNERS),	"PKCS7_GET0_SIGNERS"},
@@ -110,6 +111,7 @@ static ERR_STRING_DATA PKCS7_str_reasons[]=
 {ERR_REASON(PKCS7_R_CIPHER_HAS_NO_OBJECT_IDENTIFIER),"cipher has no object identifier"},
 {ERR_REASON(PKCS7_R_CIPHER_NOT_INITIALIZED),"cipher not initialized"},
 {ERR_REASON(PKCS7_R_CONTENT_AND_DATA_PRESENT),"content and data present"},
+{ERR_REASON(PKCS7_R_CTRL_ERROR)          ,"ctrl error"},
 {ERR_REASON(PKCS7_R_DECODE_ERROR)        ,"decode error"},
 {ERR_REASON(PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH),"decrypted key is wrong length"},
 {ERR_REASON(PKCS7_R_DECRYPT_ERROR)       ,"decrypt error"},
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 01625a2ae7..01a4030f16 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -405,6 +405,10 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 		rctx->md = p2;
 		return 1;
 
+		case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
+		case EVP_PKEY_CTRL_PKCS7_DECRYPT:
+		return 1;
+
 		default:
 		return -2;