FIPS mode RSA changes:

Check for selftest failures.

Pairwise consistency test for RSA key generation.

Use some EVP macros instead of EVP functions.

Use minimal FIPS EVP where needed.

8 files changed, 212 insertions, 15 deletions

diff --git a/crypto/rsa/Makefile b/crypto/rsa/Makefile
index d14a88b77d..69afa27fcd 100644
--- a/crypto/rsa/Makefile
+++ b/crypto/rsa/Makefile
@@ -39,7 +39,7 @@ top:
 all:	lib
 
 lib:	$(LIBOBJ)
-	$(AR) $(LIB) $(LIBOBJ)
+	$(ARX) $(LIB) $(LIBOBJ)
 	$(RANLIB) $(LIB) || echo Never mind.
 	@touch lib
 
diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index aac275fd38..47ba358eb6 100644
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -164,6 +164,8 @@ struct rsa_st
 # define OPENSSL_RSA_MAX_MODULUS_BITS	16384
 #endif
 
+#define OPENSSL_RSA_FIPS_MIN_MODULUS_BITS 1024
+
 #ifndef OPENSSL_RSA_SMALL_MODULUS_BITS
 # define OPENSSL_RSA_SMALL_MODULUS_BITS	3072
 #endif
@@ -291,6 +293,12 @@ RSA *	RSA_generate_key(int bits, unsigned long e,void
 /* New version */
 int	RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
 
+int RSA_X931_derive_ex(RSA *rsa, BIGNUM *p1, BIGNUM *p2, BIGNUM *q1, BIGNUM *q2,
+			const BIGNUM *Xp1, const BIGNUM *Xp2, const BIGNUM *Xp,
+			const BIGNUM *Xq1, const BIGNUM *Xq2, const BIGNUM *Xq,
+			const BIGNUM *e, BN_GENCB *cb);
+int RSA_X931_generate_key_ex(RSA *rsa, int bits, const BIGNUM *e, BN_GENCB *cb);
+
 int	RSA_check_key(const RSA *);
 	/* next 4 return -1 on error */
 int	RSA_public_encrypt(int flen, const unsigned char *from,
@@ -307,6 +315,17 @@ int	RSA_up_ref(RSA *r);
 
 int	RSA_flags(const RSA *r);
 
+#ifdef OPENSSL_FIPS
+RSA *FIPS_rsa_new(void);
+void FIPS_rsa_free(RSA *r);
+int FIPS_rsa_sign_ctx(RSA *rsa, EVP_MD_CTX *ctx,
+			int rsa_pad_mode, int saltlen, const EVP_MD *mgf1Hash,
+			unsigned char *sigret, unsigned int *siglen);
+int FIPS_rsa_verify_ctx(RSA *rsa, EVP_MD_CTX *ctx,
+			int rsa_pad_mode, int saltlen, const EVP_MD *mgf1Hash,
+			unsigned char *sigbuf, unsigned int siglen);
+#endif
+
 void RSA_set_default_method(const RSA_METHOD *meth);
 const RSA_METHOD *RSA_get_default_method(void);
 const RSA_METHOD *RSA_get_method(const RSA *rsa);
@@ -413,6 +432,14 @@ int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
 			const unsigned char *mHash,
 			const EVP_MD *Hash, int sLen);
 
+int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
+			const EVP_MD *Hash, const EVP_MD *mgf1Hash, 
+			const unsigned char *EM, int sLen);
+
+int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
+			const unsigned char *mHash,
+			const EVP_MD *Hash, const EVP_MD *mgf1Hash, int sLen);
+
 int RSA_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new *new_func,
 	CRYPTO_EX_dup *dup_func, CRYPTO_EX_free *free_func);
 int RSA_set_ex_data(RSA *r,int idx,void *arg);
@@ -421,6 +448,21 @@ void *RSA_get_ex_data(const RSA *r, int idx);
 RSA *RSAPublicKey_dup(RSA *rsa);
 RSA *RSAPrivateKey_dup(RSA *rsa);
 
+/* If this flag is set the RSA method is FIPS compliant and can be used
+ * in FIPS mode. This is set in the validated module method. If an
+ * application sets this flag in its own methods it is its reposibility
+ * to ensure the result is compliant.
+ */
+
+#define RSA_FLAG_FIPS_METHOD			0x0400
+
+/* If this flag is set the operations normally disabled in FIPS mode are
+ * permitted it is then the applications responsibility to ensure that the
+ * usage is compliant.
+ */
+
+#define RSA_FLAG_NON_FIPS_ALLOW			0x0400
+
 /* BEGIN ERROR CODES */
 /* The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.
@@ -432,6 +474,8 @@ void ERR_load_RSA_strings(void);
 /* Function codes. */
 #define RSA_F_CHECK_PADDING_MD				 140
 #define RSA_F_DO_RSA_PRINT				 146
+#define RSA_F_FIPS_RSA_SIGN				 149
+#define RSA_F_FIPS_RSA_VERIFY				 150
 #define RSA_F_INT_RSA_VERIFY				 145
 #define RSA_F_MEMORY_LOCK				 100
 #define RSA_F_OLD_RSA_PRIV_DECODE			 147
diff --git a/crypto/rsa/rsa_eay.c b/crypto/rsa/rsa_eay.c
index 7c941885f0..8d3107d84f 100644
--- a/crypto/rsa/rsa_eay.c
+++ b/crypto/rsa/rsa_eay.c
@@ -114,6 +114,9 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 #include <openssl/rand.h>
+#ifdef OPENSSL_FIPS
+#include <openssl/fips.h>
+#endif
 
 #ifndef RSA_NULL
 
@@ -138,7 +141,7 @@ static RSA_METHOD rsa_pkcs1_eay_meth={
 	BN_mod_exp_mont, /* XXX probably we should not use Montgomery if  e == 3 */
 	RSA_eay_init,
 	RSA_eay_finish,
-	0, /* flags */
+	RSA_FLAG_FIPS_METHOD, /* flags */
 	NULL,
 	0, /* rsa_sign */
 	0, /* rsa_verify */
@@ -158,6 +161,20 @@ static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+#ifdef OPENSSL_FIPS
+	if(FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT,FIPS_R_FIPS_SELFTEST_FAILED);
+		goto err;
+		}
+
+	if (FIPS_mode() && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL);
+		return -1;
+		}
+#endif
+
 	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
 		{
 		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
@@ -355,6 +372,20 @@ static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
 	int local_blinding = 0;
 	BN_BLINDING *blinding = NULL;
 
+#ifdef OPENSSL_FIPS
+	if(FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT,FIPS_R_FIPS_SELFTEST_FAILED);
+		goto err;
+		}
+
+	if (FIPS_mode() && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL);
+		return -1;
+		}
+#endif
+
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
 	BN_CTX_start(ctx);
 	f   = BN_CTX_get(ctx);
@@ -488,6 +519,20 @@ static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
 	int local_blinding = 0;
 	BN_BLINDING *blinding = NULL;
 
+#ifdef OPENSSL_FIPS
+	if(FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT,FIPS_R_FIPS_SELFTEST_FAILED);
+		goto err;
+		}
+
+	if (FIPS_mode() && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL);
+		return -1;
+		}
+#endif
+
 	if((ctx = BN_CTX_new()) == NULL) goto err;
 	BN_CTX_start(ctx);
 	f   = BN_CTX_get(ctx);
@@ -617,6 +662,20 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+#ifdef OPENSSL_FIPS
+	if(FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT,FIPS_R_FIPS_SELFTEST_FAILED);
+		goto err;
+		}
+
+	if (FIPS_mode() && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL);
+		return -1;
+		}
+#endif
+
 	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
 		{
 		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE);
@@ -875,6 +934,9 @@ err:
 
 static int RSA_eay_init(RSA *rsa)
 	{
+#ifdef OPENSSL_FIPS
+	FIPS_selftest_check();
+#endif
 	rsa->flags|=RSA_FLAG_CACHE_PUBLIC|RSA_FLAG_CACHE_PRIVATE;
 	return(1);
 	}
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index 293902d15e..a06d681c34 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -72,6 +72,8 @@ static ERR_STRING_DATA RSA_str_functs[]=
 	{
 {ERR_FUNC(RSA_F_CHECK_PADDING_MD),	"CHECK_PADDING_MD"},
 {ERR_FUNC(RSA_F_DO_RSA_PRINT),	"DO_RSA_PRINT"},
+{ERR_FUNC(RSA_F_FIPS_RSA_SIGN),	"FIPS_RSA_SIGN"},
+{ERR_FUNC(RSA_F_FIPS_RSA_VERIFY),	"FIPS_RSA_VERIFY"},
 {ERR_FUNC(RSA_F_INT_RSA_VERIFY),	"INT_RSA_VERIFY"},
 {ERR_FUNC(RSA_F_MEMORY_LOCK),	"MEMORY_LOCK"},
 {ERR_FUNC(RSA_F_OLD_RSA_PRIV_DECODE),	"OLD_RSA_PRIV_DECODE"},
diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 767f7ab682..b8676ad020 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -68,6 +68,77 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
+#ifdef OPENSSL_FIPS
+
+#include <openssl/fips.h>
+#include <openssl/evp.h>
+
+static int fips_rsa_pairwise_fail = 0;
+
+void FIPS_corrupt_rsa_keygen(void)
+	{
+	fips_rsa_pairwise_fail = 1;
+	}
+
+int fips_check_rsa(RSA *rsa)
+	{
+	const unsigned char tbs[] = "RSA Pairwise Check Data";
+	unsigned char *ctbuf = NULL, *ptbuf = NULL;
+	int len, ret = 0;
+	EVP_PKEY pk;
+    	pk.type = EVP_PKEY_RSA;
+    	pk.pkey.rsa = rsa;
+
+	/* Perform pairwise consistency signature test */
+	if (!fips_pkey_signature_test(&pk, tbs, -1,
+			NULL, 0, EVP_sha1(), RSA_PKCS1_PADDING, NULL)
+		|| !fips_pkey_signature_test(&pk, tbs, -1,
+			NULL, 0, EVP_sha1(), RSA_X931_PADDING, NULL)
+		|| !fips_pkey_signature_test(&pk, tbs, -1,
+			NULL, 0, EVP_sha1(), RSA_PKCS1_PSS_PADDING, NULL))
+		goto err;
+	/* Now perform pairwise consistency encrypt/decrypt test */
+	ctbuf = OPENSSL_malloc(RSA_size(rsa));
+	if (!ctbuf)
+		goto err;
+
+	len = RSA_public_encrypt(sizeof(tbs) - 1, tbs, ctbuf, rsa, RSA_PKCS1_PADDING);
+	if (len <= 0)
+		goto err;
+	/* Check ciphertext doesn't match plaintext */
+	if ((len == (sizeof(tbs) - 1)) && !memcmp(tbs, ctbuf, len))
+		goto err;
+	ptbuf = OPENSSL_malloc(RSA_size(rsa));
+
+	if (!ptbuf)
+		goto err;
+	len = RSA_private_decrypt(len, ctbuf, ptbuf, rsa, RSA_PKCS1_PADDING);
+	if (len != (sizeof(tbs) - 1))
+		goto err;
+	if (memcmp(ptbuf, tbs, len))
+		goto err;
+
+	ret = 1;
+
+	if (!ptbuf)
+		goto err;
+	
+	err:
+	if (ret == 0)
+		{
+		fips_set_selftest_fail();
+		FIPSerr(FIPS_F_FIPS_CHECK_RSA,FIPS_R_PAIRWISE_TEST_FAILED);
+		}
+
+	if (ctbuf)
+		OPENSSL_free(ctbuf);
+	if (ptbuf)
+		OPENSSL_free(ptbuf);
+
+	return ret;
+	}
+#endif
+
 static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb);
 
 /* NB: this wrapper would normally be placed in rsa_lib.c and the static
@@ -90,6 +161,20 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 	int bitsp,bitsq,ok= -1,n=0;
 	BN_CTX *ctx=NULL;
 
+#ifdef OPENSSL_FIPS
+	if(FIPS_selftest_failed())
+	    {
+	    FIPSerr(FIPS_F_RSA_BUILTIN_KEYGEN,FIPS_R_FIPS_SELFTEST_FAILED);
+	    return 0;
+	    }
+
+	if (FIPS_mode() && (bits < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
+	    {
+	    FIPSerr(FIPS_F_RSA_BUILTIN_KEYGEN,FIPS_R_KEY_TOO_SHORT);
+	    return 0;
+	    }
+#endif
+
 	ctx=BN_CTX_new();
 	if (ctx == NULL) goto err;
 	BN_CTX_start(ctx);
@@ -201,6 +286,14 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 		p = rsa->p;
 	if (!BN_mod_inverse(rsa->iqmp,rsa->q,p,ctx)) goto err;
 
+#ifdef OPENSSL_FIPS
+	if (fips_rsa_pairwise_fail)
+		BN_add_word(rsa->n, 1);
+
+	if(!fips_check_rsa(rsa))
+	    goto err;
+#endif
+
 	ok=1;
 err:
 	if (ok == -1)
diff --git a/crypto/rsa/rsa_locl.h b/crypto/rsa/rsa_locl.h
index e8ea477929..f5d2d56628 100644
--- a/crypto/rsa/rsa_locl.h
+++ b/crypto/rsa/rsa_locl.h
@@ -2,11 +2,3 @@ extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
 		unsigned char *rm, size_t *prm_len,
 		const unsigned char *sigbuf, size_t siglen,
 		RSA *rsa);
-
-int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
-			const EVP_MD *Hash, const EVP_MD *mgf1Hash, 
-			const unsigned char *EM, int sLen);
-
-int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
-			const unsigned char *mHash,
-			const EVP_MD *Hash, const EVP_MD *mgf1Hash, int sLen);
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 553d212ebe..bf8dd044b1 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -18,6 +18,8 @@
  * an equivalent notion.
  */
 
+#define OPENSSL_FIPSEVP
+
 
 #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1)
 #include <stdio.h>
@@ -194,7 +196,7 @@ int PKCS1_MGF1(unsigned char *mask, long len,
 	int rv = -1;
 
 	EVP_MD_CTX_init(&c);
-	mdlen = EVP_MD_size(dgst);
+	mdlen = M_EVP_MD_size(dgst);
 	if (mdlen < 0)
 		goto err;
 	for (i = 0; outlen < len; i++)
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 4f87a2939d..794de9dff6 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -56,6 +56,8 @@
  *
  */
 
+#define OPENSSL_FIPSEVP
+
 #include <stdio.h>
 #include "cryptlib.h"
 #include <openssl/bn.h>
@@ -93,7 +95,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
 	if (mgf1Hash == NULL)
 		mgf1Hash = Hash;
 
-	hLen = EVP_MD_size(Hash);
+	hLen = M_EVP_MD_size(Hash);
 	if (hLen < 0)
 		goto err;
 	/*
@@ -166,7 +168,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
 		if (!EVP_DigestUpdate(&ctx, DB + i, maskedDBLen - i))
 			goto err;
 		}
-	if (!EVP_DigestFinal(&ctx, H_, NULL))
+	if (!EVP_DigestFinal_ex(&ctx, H_, NULL))
 		goto err;
 	if (memcmp(H_, H, hLen))
 		{
@@ -205,7 +207,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
 	if (mgf1Hash == NULL)
 		mgf1Hash = Hash;
 
-	hLen = EVP_MD_size(Hash);
+	hLen = M_EVP_MD_size(Hash);
 	if (hLen < 0)
 		goto err;
 	/*
@@ -260,7 +262,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
 		goto err;
 	if (sLen && !EVP_DigestUpdate(&ctx, salt, sLen))
 		goto err;
-	if (!EVP_DigestFinal(&ctx, H, NULL))
+	if (!EVP_DigestFinal_ex(&ctx, H, NULL))
 		goto err;
 	EVP_MD_CTX_cleanup(&ctx);