If we're going to return errors (no matter how stupid), then we should

test for them!

4 files changed, 21 insertions, 16 deletions

diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index 91cd4198c7..cf74343657 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -448,7 +448,6 @@ void ERR_load_RSA_strings(void);
 
 /* Reason codes. */
 #define RSA_R_ALGORITHM_MISMATCH			 100
-#define RSA_R_BAD_ARGUMENT				 149
 #define RSA_R_BAD_E_VALUE				 101
 #define RSA_R_BAD_FIXED_HEADER_DECRYPT			 102
 #define RSA_R_BAD_PAD_BYTE_COUNT			 103
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index a53c5f6bff..cf9f1106b0 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -124,7 +124,6 @@ static ERR_STRING_DATA RSA_str_functs[]=
 static ERR_STRING_DATA RSA_str_reasons[]=
 	{
 {ERR_REASON(RSA_R_ALGORITHM_MISMATCH)    ,"algorithm mismatch"},
-{ERR_REASON(RSA_R_BAD_ARGUMENT)          ,"bad argument"},
 {ERR_REASON(RSA_R_BAD_E_VALUE)           ,"bad e value"},
 {ERR_REASON(RSA_R_BAD_FIXED_HEADER_DECRYPT),"bad fixed header decrypt"},
 {ERR_REASON(RSA_R_BAD_PAD_BYTE_COUNT)    ,"bad pad byte count"},
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 3652677a99..70bacf850e 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -28,7 +28,7 @@
 #include <openssl/rand.h>
 #include <openssl/sha.h>
 
-int MGF1(unsigned char *mask, long len,
+static int MGF1(unsigned char *mask, long len,
 	const unsigned char *seed, long seedlen);
 
 int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
@@ -76,11 +76,13 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
 	   20);
 #endif
 
-	MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
+	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH) < 0)
+		return 0;
 	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
 		db[i] ^= dbmask[i];
 
-	MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);
+	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH) < 0)
+		return 0;
 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
 		seed[i] ^= seedmask[i];
 
@@ -133,11 +135,13 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 
 	maskeddb = padded_from + SHA_DIGEST_LENGTH;
 
-	MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
+	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
+		return -1;
 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
 		seed[i] ^= padded_from[i];
   
-	MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
+	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
+		return -1;
 	for (i = 0; i < dblen; i++)
 		db[i] ^= maskeddb[i];
 
@@ -188,6 +192,8 @@ int PKCS1_MGF1(unsigned char *mask, long len,
 
 	EVP_MD_CTX_init(&c);
 	mdlen = EVP_MD_size(dgst);
+	if (mdlen < 0)
+		return -1;
 	for (i = 0; outlen < len; i++)
 		{
 		cnt[0] = (unsigned char)((i >> 24) & 255);
@@ -213,7 +219,8 @@ int PKCS1_MGF1(unsigned char *mask, long len,
 	return 0;
 	}
 
-int MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
+static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
+		 long seedlen)
 	{
 	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
 	}
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 2e44194bdc..775c36114f 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -81,13 +81,9 @@ int RSA_verify_PKCS1_PSS(RSA *rsa, const unsigned char *mHash,
 	EVP_MD_CTX ctx;
 	unsigned char H_[EVP_MAX_MD_SIZE];
 
-	if (Hash == NULL)
-		{
-		RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS, RSA_R_BAD_ARGUMENT);
-		goto err;
-		}
-
 	hLen = EVP_MD_size(Hash);
+	if (hLen < 0)
+		goto err;
 	/*
 	 * Negative sLen has special meanings:
 	 *	-1	sLen == hLen
@@ -132,7 +128,8 @@ int RSA_verify_PKCS1_PSS(RSA *rsa, const unsigned char *mHash,
 		RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS, ERR_R_MALLOC_FAILURE);
 		goto err;
 		}
-	PKCS1_MGF1(DB, maskedDBLen, H, hLen, Hash);
+	if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, Hash) < 0)
+		goto err;
 	for (i = 0; i < maskedDBLen; i++)
 		DB[i] ^= EM[i];
 	if (MSBits)
@@ -183,6 +180,8 @@ int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
 	EVP_MD_CTX ctx;
 
 	hLen = EVP_MD_size(Hash);
+	if (hLen < 0)
+		goto err;
 	/*
 	 * Negative sLen has special meanings:
 	 *	-1	sLen == hLen
@@ -238,7 +237,8 @@ int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
 	EVP_MD_CTX_cleanup(&ctx);
 
 	/* Generate dbMask in place then perform XOR on it */
-	PKCS1_MGF1(EM, maskedDBLen, H, hLen, Hash);
+	if (PKCS1_MGF1(EM, maskedDBLen, H, hLen, Hash))
+		goto err;
 
 	p = EM;