Support for digest signing and X931 in rsa_pkey_meth.

7 files changed, 174 insertions, 11 deletions

diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h
index 07bcb50ad0..e09c55377c 100644
--- a/crypto/evp/evp.h
+++ b/crypto/evp/evp.h
@@ -914,6 +914,10 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
 #define EVP_PKEY_OP_ENCRYPT		8
 #define EVP_PKEY_OP_DECRYPT		9
 
+#define EVP_PKEY_CTRL_MD_NID		1
+
+#define EVP_PKEY_ALG_CTRL		0x1000
+
 const EVP_PKEY_METHOD *EVP_PKEY_meth_find(int type, ENGINE *e);
 EVP_PKEY_CTX *EVP_PKEY_CTX_new(EVP_PKEY *pkey);
 void EVP_PKEY_CTX_free(EVP_PKEY_CTX *ctx);
@@ -1024,6 +1028,7 @@ void ERR_load_EVP_strings(void);
 #define EVP_R_EXPECTING_A_EC_KEY			 142
 #define EVP_R_INITIALIZATION_ERROR			 134
 #define EVP_R_INPUT_NOT_INITIALIZED			 111
+#define EVP_R_INVALID_DIGEST				 152
 #define EVP_R_INVALID_KEY_LENGTH			 130
 #define EVP_R_INVALID_OPERATION				 148
 #define EVP_R_IV_TOO_LARGE				 102
diff --git a/crypto/evp/evp_err.c b/crypto/evp/evp_err.c
index 702331a4a9..0f14fa8a7d 100644
--- a/crypto/evp/evp_err.c
+++ b/crypto/evp/evp_err.c
@@ -143,6 +143,7 @@ static ERR_STRING_DATA EVP_str_reasons[]=
 {ERR_REASON(EVP_R_EXPECTING_A_EC_KEY)    ,"expecting a ec key"},
 {ERR_REASON(EVP_R_INITIALIZATION_ERROR)  ,"initialization error"},
 {ERR_REASON(EVP_R_INPUT_NOT_INITIALIZED) ,"input not initialized"},
+{ERR_REASON(EVP_R_INVALID_DIGEST)        ,"invalid digest"},
 {ERR_REASON(EVP_R_INVALID_KEY_LENGTH)    ,"invalid key length"},
 {ERR_REASON(EVP_R_INVALID_OPERATION)     ,"invalid operation"},
 {ERR_REASON(EVP_R_IV_TOO_LARGE)          ,"iv too large"},
diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c
index 7c36395696..a705901c70 100644
--- a/crypto/evp/pmeth_lib.c
+++ b/crypto/evp/pmeth_lib.c
@@ -179,5 +179,16 @@ int EVP_PKEY_CTX_ctrl_str(EVP_PKEY_CTX *ctx,
 		EVPerr(EVP_F_EVP_PKEY_CTX_CTRL, EVP_R_COMMAND_NOT_SUPPORTED);
 		return -2;
 		}
+	if (!strcmp(name, "digest"))
+		{
+		const EVP_MD *md;
+		if (!value || !(md = EVP_get_digestbyname(value)))
+			{
+			EVPerr(EVP_F_EVP_PKEY_CTX_CTRL, EVP_R_INVALID_DIGEST);
+			return 0;
+			}
+		return EVP_PKEY_CTX_ctrl(ctx, -1, -1, EVP_PKEY_CTRL_MD_NID,
+					EVP_MD_type(md), NULL);
+		}
 	return ctx->pmeth->ctrl_str(ctx, name, value);
 	}
diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index 09db1c88de..a7a06b4acb 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -196,7 +196,7 @@ struct rsa_st
 	EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, -1, EVP_PKEY_CTRL_RSA_PADDING, \
 				pad, NULL)
 
-#define EVP_PKEY_CTRL_RSA_PADDING	1
+#define EVP_PKEY_CTRL_RSA_PADDING	(EVP_PKEY_ALG_CTRL + 1)
 
 #define RSA_PKCS1_PADDING	1
 #define RSA_SSLV23_PADDING	2
@@ -349,7 +349,9 @@ void ERR_load_RSA_strings(void);
 /* Error codes for the RSA functions. */
 
 /* Function codes. */
+#define RSA_F_CHECK_PADDING_NID				 140
 #define RSA_F_MEMORY_LOCK				 100
+#define RSA_F_PKEY_RSA_VERIFYRECOVER			 141
 #define RSA_F_RSA_BUILTIN_KEYGEN			 129
 #define RSA_F_RSA_CHECK_KEY				 123
 #define RSA_F_RSA_EAY_PRIVATE_DECRYPT			 101
@@ -409,10 +411,13 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_DMQ1_NOT_CONGRUENT_TO_D			 125
 #define RSA_R_D_E_NOT_CONGRUENT_TO_1			 123
 #define RSA_R_FIRST_OCTET_INVALID			 133
+#define RSA_R_INVALID_DIGEST				 105
 #define RSA_R_INVALID_HEADER				 137
 #define RSA_R_INVALID_MESSAGE_LENGTH			 131
 #define RSA_R_INVALID_PADDING				 138
+#define RSA_R_INVALID_PADDING_MODE			 141
 #define RSA_R_INVALID_TRAILER				 139
+#define RSA_R_INVALID_X931_DIGEST			 142
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q			 126
 #define RSA_R_KEY_SIZE_TOO_SMALL			 120
 #define RSA_R_LAST_OCTET_INVALID			 134
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index fc43065317..f403d0cedf 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -70,7 +70,9 @@
 
 static ERR_STRING_DATA RSA_str_functs[]=
 	{
+{ERR_FUNC(RSA_F_CHECK_PADDING_NID),	"CHECK_PADDING_NID"},
 {ERR_FUNC(RSA_F_MEMORY_LOCK),	"MEMORY_LOCK"},
+{ERR_FUNC(RSA_F_PKEY_RSA_VERIFYRECOVER),	"PKEY_RSA_VERIFYRECOVER"},
 {ERR_FUNC(RSA_F_RSA_BUILTIN_KEYGEN),	"RSA_BUILTIN_KEYGEN"},
 {ERR_FUNC(RSA_F_RSA_CHECK_KEY),	"RSA_check_key"},
 {ERR_FUNC(RSA_F_RSA_EAY_PRIVATE_DECRYPT),	"RSA_EAY_PRIVATE_DECRYPT"},
@@ -100,7 +102,7 @@ static ERR_STRING_DATA RSA_str_functs[]=
 {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_SSLV23),	"RSA_padding_check_SSLv23"},
 {ERR_FUNC(RSA_F_RSA_PADDING_CHECK_X931),	"RSA_padding_check_X931"},
 {ERR_FUNC(RSA_F_RSA_PRINT),	"RSA_print"},
-{ERR_FUNC(RSA_F_RSA_PRINT_FP),	"RSA_print_fp"},
+{ERR_FUNC(RSA_F_RSA_PRINT_FP),	"RSA_PRINT_FP"},
 {ERR_FUNC(RSA_F_RSA_PRIV_DECODE),	"RSA_PRIV_DECODE"},
 {ERR_FUNC(RSA_F_RSA_PRIV_ENCODE),	"RSA_PRIV_ENCODE"},
 {ERR_FUNC(RSA_F_RSA_PUB_DECODE),	"RSA_PUB_DECODE"},
@@ -133,10 +135,13 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {ERR_REASON(RSA_R_DMQ1_NOT_CONGRUENT_TO_D),"dmq1 not congruent to d"},
 {ERR_REASON(RSA_R_D_E_NOT_CONGRUENT_TO_1),"d e not congruent to 1"},
 {ERR_REASON(RSA_R_FIRST_OCTET_INVALID)   ,"first octet invalid"},
+{ERR_REASON(RSA_R_INVALID_DIGEST)        ,"invalid digest"},
 {ERR_REASON(RSA_R_INVALID_HEADER)        ,"invalid header"},
 {ERR_REASON(RSA_R_INVALID_MESSAGE_LENGTH),"invalid message length"},
 {ERR_REASON(RSA_R_INVALID_PADDING)       ,"invalid padding"},
+{ERR_REASON(RSA_R_INVALID_PADDING_MODE)  ,"invalid padding mode"},
 {ERR_REASON(RSA_R_INVALID_TRAILER)       ,"invalid trailer"},
+{ERR_REASON(RSA_R_INVALID_X931_DIGEST)   ,"invalid x931 digest"},
 {ERR_REASON(RSA_R_IQMP_NOT_INVERSE_OF_Q) ,"iqmp not inverse of q"},
 {ERR_REASON(RSA_R_KEY_SIZE_TOO_SMALL)    ,"key size too small"},
 {ERR_REASON(RSA_R_LAST_OCTET_INVALID)    ,"last octet invalid"},
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 3552fce8d9..a93adb75b9 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -60,8 +60,14 @@
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
 #include <openssl/rsa.h>
+#include <openssl/evp.h>
 #include "evp_locl.h"
 
+extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
+		unsigned char *rm, unsigned int *prm_len,
+		unsigned char *sigbuf, unsigned int siglen,
+		RSA *rsa);
+
 /* RSA pkey context structure */
 
 typedef struct
@@ -71,6 +77,10 @@ typedef struct
 	BIGNUM *pub_exp;
 	/* RSA padding mode */
 	int pad_mode;
+	/* nid for message digest */
+	int md_nid;
+	/* Temp buffer */
+	unsigned char *tbuf;
 	} RSA_PKEY_CTX;
 
 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
@@ -82,7 +92,21 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 	rctx->nbits = 1024;
 	rctx->pub_exp = NULL;
 	rctx->pad_mode = RSA_PKCS1_PADDING;
+	rctx->md_nid = NID_undef;
+	rctx->tbuf = NULL;
+
 	ctx->data = rctx;
+	
+	return 1;
+	}
+
+static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
+	{
+	if (ctx->tbuf)
+		return 1;
+	ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
+	if (!ctx->tbuf)
+		return 0;
 	return 1;
 	}
 
@@ -102,7 +126,31 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 	{
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
-	ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
+
+	if (rctx->md_nid != NID_undef)
+		{
+
+		if (rctx->pad_mode == RSA_X931_PADDING)
+			{
+			if (!setup_tbuf(rctx, ctx))
+				return -1;
+			memcpy(rctx->tbuf, tbs, tbslen);
+			rctx->tbuf[tbslen] = RSA_X931_hash_id(rctx->md_nid);
+			ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
+						sig, ctx->pkey->pkey.rsa,
+						RSA_X931_PADDING);
+			}
+		else if (rctx->pad_mode == RSA_PKCS1_PADDING)
+			{
+			unsigned int sltmp;
+			ret = RSA_sign(rctx->md_nid, tbs, tbslen, sig, &sltmp,
+							ctx->pkey->pkey.rsa);
+			}
+		else
+			return -1;
+		}
+	else
+		ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 							rctx->pad_mode);
 	if (ret < 0)
 		return ret;
@@ -117,7 +165,38 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 	{
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
-	ret = RSA_public_decrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
+
+	if (rctx->md_nid != NID_undef)
+		{
+		if (rctx->pad_mode == RSA_X931_PADDING)
+			{
+			if (!setup_tbuf(rctx, ctx))
+				return -1;
+			ret = RSA_private_encrypt(tbslen, tbs,
+						rctx->tbuf, ctx->pkey->pkey.rsa,
+						RSA_X931_PADDING);
+			if (ret < 1)
+				return 0;
+			if (rctx->tbuf[ret] != RSA_X931_hash_id(rctx->md_nid))
+				{
+				RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
+						RSA_R_ALGORITHM_MISMATCH);
+				return 0;
+				}
+			ret--;
+			memcpy(sig, rctx->tbuf, ret);
+			}
+		else if (rctx->pad_mode == RSA_PKCS1_PADDING)
+			{
+			unsigned int sltmp;
+			ret = int_rsa_verify(rctx->md_nid, NULL, 0, sig, &sltmp,
+					tbs, tbslen, ctx->pkey->pkey.rsa);
+			}
+		else
+			return -1;
+		}
+	else
+		ret = RSA_public_decrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 							rctx->pad_mode);
 	if (ret < 0)
 		return ret;
@@ -151,23 +230,55 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 	return 1;
 	}
 
+static int check_padding_nid(int nid, int padding)
+	{
+	if (nid == NID_undef)
+		return 1;
+	if (padding == RSA_NO_PADDING)
+		{
+		RSAerr(RSA_F_CHECK_PADDING_NID, RSA_R_INVALID_PADDING_MODE);
+		return 0;
+		}
+
+	if (padding == RSA_X931_PADDING)
+		{
+		if (RSA_X931_hash_id(nid) == -1)
+			{
+			RSAerr(RSA_F_CHECK_PADDING_NID,
+						RSA_R_INVALID_X931_DIGEST);
+			return 0;
+			}
+		return 1;
+		}
+
+	return 1;
+	}
+			
+
 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 	{
 	RSA_PKEY_CTX *rctx = ctx->data;
 	switch (type)
 		{
-
 		case EVP_PKEY_CTRL_RSA_PADDING:
 		/* TODO: add PSS support */
 		if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_X931_PADDING))
 			{
 			if (ctx->operation == EVP_PKEY_OP_KEYGEN)
 				return -2;
+			if (!check_padding_nid(rctx->md_nid, p1))
+				return 0;
 			rctx->pad_mode = p1;
 			return 1;
 			}
 		return -2;
 
+		case EVP_PKEY_CTRL_MD_NID:
+		if (!check_padding_nid(p1, rctx->pad_mode))
+			return 0;
+		rctx->md_nid = p1;
+		return 1;
+
 		default:
 		return -2;
 
diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c
index 230ec6d7ea..91f03406d0 100644
--- a/crypto/rsa/rsa_sign.c
+++ b/crypto/rsa/rsa_sign.c
@@ -142,8 +142,10 @@ int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
 	return(ret);
 	}
 
-int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
-	     unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
+int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
+		unsigned char *rm, unsigned int *prm_len,
+		unsigned char *sigbuf, unsigned int siglen,
+		RSA *rsa)
 	{
 	int i,ret=0,sigtype;
 	unsigned char *s;
@@ -155,10 +157,14 @@ int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
 		return(0);
 		}
 
-	if((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify)
+	if((dtype == NID_md5_sha1) && rm)
 		{
-		return rsa->meth->rsa_verify(dtype, m, m_len,
-			sigbuf, siglen, rsa);
+		i = RSA_public_decrypt((int)siglen,
+					sigbuf,rm,rsa,RSA_PKCS1_PADDING);
+		if (i <= 0)
+			return 0;
+		*prm_len = i;
+		return 1;
 		}
 
 	s=(unsigned char *)OPENSSL_malloc((unsigned int)siglen);
@@ -212,7 +218,13 @@ int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
 				goto err;
 				}
 			}
-		if (	((unsigned int)sig->digest->length != m_len) ||
+		if (rm)
+			{
+			memcpy(rm, sig->digest->data, sig->digest->length);
+			*prm_len = sig->digest->length;
+			ret = 1;
+			}
+		else if (((unsigned int)sig->digest->length != m_len) ||
 			(memcmp(m,sig->digest->data,m_len) != 0))
 			{
 			RSAerr(RSA_F_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
@@ -230,3 +242,16 @@ err:
 	return(ret);
 	}
 
+int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
+		unsigned char *sigbuf, unsigned int siglen,
+		RSA *rsa)
+	{
+
+	if((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify)
+		{
+		return rsa->meth->rsa_verify(dtype, m, m_len,
+			sigbuf, siglen, rsa);
+		}
+
+	return int_rsa_verify(dtype, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
+	}