Exetended OAEP support.

Extend OAEP support. Generalise the OAEP padding functions to support
arbitrary digests. Extend EVP_PKEY RSA method to handle the new OAEP
padding functions and add ctrls to set the additional parameters.

1 files changed, 116 insertions, 5 deletions

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 2b185f0acc..50e0d5c550 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -63,6 +63,7 @@
 #include <openssl/rsa.h>
 #include <openssl/bn.h>
 #include <openssl/evp.h>
+#include <openssl/x509v3.h>
 #ifndef OPENSSL_NO_CMS
 #include <openssl/cms.h>
 #endif
@@ -84,10 +85,13 @@ typedef struct
 	const EVP_MD *md;
 	/* message digest for MGF1 */
 	const EVP_MD *mgf1md;
-	/* PSS/OAEP salt length */
+	/* PSS salt length */
 	int saltlen;
 	/* Temp buffer */
 	unsigned char *tbuf;
+	/* OAEP label */
+	unsigned char *oaep_label;
+	size_t oaep_labellen;
 	} RSA_PKEY_CTX;
 
 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
@@ -105,6 +109,9 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 
 	rctx->saltlen = -2;
 
+	rctx->oaep_label = NULL;
+	rctx->oaep_labellen = 0;
+
 	ctx->data = rctx;
 	ctx->keygen_info = rctx->gentmp;
 	ctx->keygen_info_count = 2;
@@ -128,6 +135,17 @@ static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 		}
 	dctx->pad_mode = sctx->pad_mode;
 	dctx->md = sctx->md;
+	dctx->mgf1md = sctx->mgf1md;
+	if (sctx->oaep_label)
+		{
+		if (dctx->oaep_label)
+			OPENSSL_free(dctx->oaep_label);
+		dctx->oaep_label = BUF_memdup(sctx->oaep_label,
+						sctx->oaep_labellen);
+		if (!dctx->oaep_label)
+			return 0;
+		dctx->oaep_labellen = sctx->oaep_labellen;
+		}
 	return 1;
 	}
 
@@ -150,6 +168,8 @@ static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 			BN_free(rctx->pub_exp);
 		if (rctx->tbuf)
 			OPENSSL_free(rctx->tbuf);
+		if (rctx->oaep_label)
+			OPENSSL_free(rctx->oaep_label);
 		OPENSSL_free(rctx);
 		}
 	}
@@ -336,7 +356,6 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 	return 1;
 			
 	}
-	
 
 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
 					unsigned char *out, size_t *outlen,
@@ -344,7 +363,23 @@ static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
 	{
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
-	ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
+		{
+		int klen = RSA_size(ctx->pkey->pkey.rsa);
+		if (!setup_tbuf(rctx, ctx))
+			return -1;
+		if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
+							in, inlen,
+							rctx->oaep_label,
+							rctx->oaep_labellen,
+							rctx->md, rctx->mgf1md))
+			return -1;
+		ret = RSA_public_encrypt(klen, rctx->tbuf, out,
+						ctx->pkey->pkey.rsa,
+						RSA_NO_PADDING);
+		}
+	else
+		ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 							rctx->pad_mode);
 	if (ret < 0)
 		return ret;
@@ -358,7 +393,29 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
 	{
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
-	ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
+		{
+		int i;
+		if (!setup_tbuf(rctx, ctx))
+			return -1;
+		ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
+							ctx->pkey->pkey.rsa,
+							RSA_NO_PADDING);
+		if (ret <= 0)
+			return ret;
+		for (i = 0; i < ret; i++)
+			{
+			if (rctx->tbuf[i])
+				break;
+			}
+		ret = RSA_padding_check_PKCS1_OAEP_mgf1(out,ret,rctx->tbuf + i,
+							ret - i, ret,
+							rctx->oaep_label,
+							rctx->oaep_labellen,
+							rctx->md, rctx->mgf1md);
+		}
+	else 
+		ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 							rctx->pad_mode);
 	if (ret < 0)
 		return ret;
@@ -458,9 +515,19 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 		case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 		if (!p2)
 			return -2;
+		BN_free(rctx->pub_exp);
 		rctx->pub_exp = p2;
 		return 1;
 
+		case EVP_PKEY_CTRL_RSA_OAEP_MD:
+		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
+			{
+			RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
+			return 0;
+			}
+		rctx->md = p2;
+		return 1;
+
 		case EVP_PKEY_CTRL_MD:
 		if (!check_padding_md(p2, rctx->pad_mode))
 			return 0;
@@ -469,7 +536,8 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 
 		case EVP_PKEY_CTRL_RSA_MGF1_MD:
 		case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
-		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
+		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
+			&& rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
 			{
 			RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
 			return -2;
@@ -485,6 +553,12 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 			rctx->mgf1md = p2;
 		return 1;
 
+		case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
+		OPENSSL_free(rctx->oaep_label);
+		rctx->oaep_label = p2;
+		rctx->oaep_labellen = p1;
+		return 1;
+
 		case EVP_PKEY_CTRL_DIGESTINIT:
 		case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 		case EVP_PKEY_CTRL_PKCS7_DECRYPT:
@@ -577,6 +651,43 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 		return ret;
 		}
 
+	if (!strcmp(type, "rsa_mgf1_md"))
+		{
+		const EVP_MD *md;
+		if (!(md = EVP_get_digestbyname(value)))
+			{
+			RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+						RSA_R_INVALID_DIGEST);
+			return 0;
+			}
+		return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+		}
+
+	if (!strcmp(type, "rsa_oaep_md"))
+		{
+		const EVP_MD *md;
+		if (!(md = EVP_get_digestbyname(value)))
+			{
+			RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+						RSA_R_INVALID_DIGEST);
+			return 0;
+			}
+		return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
+		}
+	if (!strcmp(type, "rsa_oaep_label"))
+		{
+		unsigned char *lab;
+		long lablen;
+		int ret;
+		lab = string_to_hex(value, &lablen);
+		if (!lab)
+			return 0;
+		ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
+		if (ret <= 0)
+			OPENSSL_free(lab);
+		return ret;
+		}
+
 	return -2;
 	}