Store digests as EVP_MD instead of a NID.

Add digest size sanity checks.

4 files changed, 47 insertions, 19 deletions

diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index a7a06b4acb..e9f87694de 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -351,6 +351,7 @@ void ERR_load_RSA_strings(void);
 /* Function codes. */
 #define RSA_F_CHECK_PADDING_NID				 140
 #define RSA_F_MEMORY_LOCK				 100
+#define RSA_F_PKEY_RSA_SIGN				 142
 #define RSA_F_PKEY_RSA_VERIFYRECOVER			 141
 #define RSA_F_RSA_BUILTIN_KEYGEN			 129
 #define RSA_F_RSA_CHECK_KEY				 123
@@ -412,6 +413,7 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_D_E_NOT_CONGRUENT_TO_1			 123
 #define RSA_R_FIRST_OCTET_INVALID			 133
 #define RSA_R_INVALID_DIGEST				 105
+#define RSA_R_INVALID_DIGEST_LENGTH			 143
 #define RSA_R_INVALID_HEADER				 137
 #define RSA_R_INVALID_MESSAGE_LENGTH			 131
 #define RSA_R_INVALID_PADDING				 138
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index f403d0cedf..9afd099f47 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -72,6 +72,7 @@ static ERR_STRING_DATA RSA_str_functs[]=
 	{
 {ERR_FUNC(RSA_F_CHECK_PADDING_NID),	"CHECK_PADDING_NID"},
 {ERR_FUNC(RSA_F_MEMORY_LOCK),	"MEMORY_LOCK"},
+{ERR_FUNC(RSA_F_PKEY_RSA_SIGN),	"PKEY_RSA_SIGN"},
 {ERR_FUNC(RSA_F_PKEY_RSA_VERIFYRECOVER),	"PKEY_RSA_VERIFYRECOVER"},
 {ERR_FUNC(RSA_F_RSA_BUILTIN_KEYGEN),	"RSA_BUILTIN_KEYGEN"},
 {ERR_FUNC(RSA_F_RSA_CHECK_KEY),	"RSA_check_key"},
@@ -136,6 +137,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {ERR_REASON(RSA_R_D_E_NOT_CONGRUENT_TO_1),"d e not congruent to 1"},
 {ERR_REASON(RSA_R_FIRST_OCTET_INVALID)   ,"first octet invalid"},
 {ERR_REASON(RSA_R_INVALID_DIGEST)        ,"invalid digest"},
+{ERR_REASON(RSA_R_INVALID_DIGEST_LENGTH) ,"invalid digest length"},
 {ERR_REASON(RSA_R_INVALID_HEADER)        ,"invalid header"},
 {ERR_REASON(RSA_R_INVALID_MESSAGE_LENGTH),"invalid message length"},
 {ERR_REASON(RSA_R_INVALID_PADDING)       ,"invalid padding"},
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index a0db5ee2fa..001dbd0bad 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -78,7 +78,7 @@ typedef struct
 	/* RSA padding mode */
 	int pad_mode;
 	/* nid for message digest */
-	int md_nid;
+	const EVP_MD *md;
 	/* Temp buffer */
 	unsigned char *tbuf;
 	} RSA_PKEY_CTX;
@@ -92,7 +92,7 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 	rctx->nbits = 1024;
 	rctx->pub_exp = NULL;
 	rctx->pad_mode = RSA_PKCS1_PADDING;
-	rctx->md_nid = NID_undef;
+	rctx->md = NULL;
 	rctx->tbuf = NULL;
 
 	ctx->data = rctx;
@@ -129,15 +129,21 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
 
-	if (rctx->md_nid != NID_undef)
+	if (rctx->md)
 		{
-
+		if (tbslen != EVP_MD_size(rctx->md))
+			{
+			RSAerr(RSA_F_PKEY_RSA_SIGN,
+					RSA_R_INVALID_DIGEST_LENGTH);
+			return -1;
+			}
 		if (rctx->pad_mode == RSA_X931_PADDING)
 			{
 			if (!setup_tbuf(rctx, ctx))
 				return -1;
 			memcpy(rctx->tbuf, tbs, tbslen);
-			rctx->tbuf[tbslen] = RSA_X931_hash_id(rctx->md_nid);
+			rctx->tbuf[tbslen] =
+				RSA_X931_hash_id(EVP_MD_type(rctx->md));
 			ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 						sig, ctx->pkey->pkey.rsa,
 						RSA_X931_PADDING);
@@ -145,7 +151,8 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 		else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 			{
 			unsigned int sltmp;
-			ret = RSA_sign(rctx->md_nid, tbs, tbslen, sig, &sltmp,
+			ret = RSA_sign(EVP_MD_type(rctx->md),
+						tbs, tbslen, sig, &sltmp,
 							ctx->pkey->pkey.rsa);
 			}
 		else
@@ -168,7 +175,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 	int ret;
 	RSA_PKEY_CTX *rctx = ctx->data;
 
-	if (rctx->md_nid != NID_undef)
+	if (rctx->md)
 		{
 		if (rctx->pad_mode == RSA_X931_PADDING)
 			{
@@ -180,18 +187,26 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 			if (ret < 1)
 				return 0;
 			ret--;
-			if (rctx->tbuf[ret] != RSA_X931_hash_id(rctx->md_nid))
+			if (rctx->tbuf[ret] !=
+				RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 				{
 				RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 						RSA_R_ALGORITHM_MISMATCH);
 				return 0;
 				}
+			if (ret != EVP_MD_size(rctx->md))
+				{
+				RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
+					RSA_R_INVALID_DIGEST_LENGTH);
+				return 0;
+				}
 			memcpy(sig, rctx->tbuf, ret);
 			}
 		else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 			{
 			unsigned int sltmp;
-			ret = int_rsa_verify(rctx->md_nid, NULL, 0, sig, &sltmp,
+			ret = int_rsa_verify(EVP_MD_type(rctx->md),
+						NULL, 0, sig, &sltmp,
 					tbs, tbslen, ctx->pkey->pkey.rsa);
 			}
 		else
@@ -232,9 +247,9 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 	return 1;
 	}
 
-static int check_padding_nid(int nid, int padding)
+static int check_padding_md(const EVP_MD *md, int padding)
 	{
-	if (nid == NID_undef)
+	if (!md)
 		return 1;
 	if (padding == RSA_NO_PADDING)
 		{
@@ -244,7 +259,7 @@ static int check_padding_nid(int nid, int padding)
 
 	if (padding == RSA_X931_PADDING)
 		{
-		if (RSA_X931_hash_id(nid) == -1)
+		if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 			{
 			RSAerr(RSA_F_CHECK_PADDING_NID,
 						RSA_R_INVALID_X931_DIGEST);
@@ -268,17 +283,17 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 			{
 			if (ctx->operation == EVP_PKEY_OP_KEYGEN)
 				return -2;
-			if (!check_padding_nid(rctx->md_nid, p1))
+			if (!check_padding_md(rctx->md, p1))
 				return 0;
 			rctx->pad_mode = p1;
 			return 1;
 			}
 		return -2;
 
-		case EVP_PKEY_CTRL_MD_NID:
-		if (!check_padding_nid(p1, rctx->pad_mode))
+		case EVP_PKEY_CTRL_MD:
+		if (!check_padding_md(p2, rctx->pad_mode))
 			return 0;
-		rctx->md_nid = p1;
+		rctx->md = p2;
 		return 1;
 
 		default:
diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c
index 91f03406d0..4d48164b77 100644
--- a/crypto/rsa/rsa_sign.c
+++ b/crypto/rsa/rsa_sign.c
@@ -220,9 +220,18 @@ int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
 			}
 		if (rm)
 			{
-			memcpy(rm, sig->digest->data, sig->digest->length);
-			*prm_len = sig->digest->length;
-			ret = 1;
+			const EVP_MD *md;
+			md = EVP_get_digestbynid(dtype);
+			if (md && (EVP_MD_size(md) != sig->digest->length))
+				RSAerr(RSA_F_RSA_VERIFY,
+						RSA_R_INVALID_DIGEST_LENGTH);
+			else
+				{
+				memcpy(rm, sig->digest->data,
+							sig->digest->length);
+				*prm_len = sig->digest->length;
+				ret = 1;
+				}
 			}
 		else if (((unsigned int)sig->digest->length != m_len) ||
 			(memcmp(m,sig->digest->data,m_len) != 0))