Return error codes for selftest failure instead of hard assertion errors.

7 files changed, 65 insertions, 11 deletions

diff --git a/crypto/fips_err.h b/crypto/fips_err.h
index 5555f2d50b..dfb24ca6fa 100644
--- a/crypto/fips_err.h
+++ b/crypto/fips_err.h
@@ -83,8 +83,12 @@ static ERR_STRING_DATA FIPS_str_functs[]=
 {ERR_FUNC(FIPS_F_FIPS_CHECK_INCORE_FINGERPRINT),	"FIPS_check_incore_fingerprint"},
 {ERR_FUNC(FIPS_F_FIPS_CHECK_RSA),	"fips_check_rsa"},
 {ERR_FUNC(FIPS_F_FIPS_CHECK_RSA_PRNG),	"fips_check_rsa_prng"},
+{ERR_FUNC(FIPS_F_FIPS_CIPHER),	"FIPS_CIPHER"},
 {ERR_FUNC(FIPS_F_FIPS_CIPHERINIT),	"FIPS_CIPHERINIT"},
+{ERR_FUNC(FIPS_F_FIPS_CIPHER_CTX_CTRL),	"FIPS_CIPHER_CTX_CTRL"},
+{ERR_FUNC(FIPS_F_FIPS_DIGESTFINAL),	"FIPS_DIGESTFINAL"},
 {ERR_FUNC(FIPS_F_FIPS_DIGESTINIT),	"FIPS_DIGESTINIT"},
+{ERR_FUNC(FIPS_F_FIPS_DIGESTUPDATE),	"FIPS_DIGESTUPDATE"},
 {ERR_FUNC(FIPS_F_FIPS_DRBG_BYTES),	"FIPS_DRBG_BYTES"},
 {ERR_FUNC(FIPS_F_FIPS_DRBG_CHECK),	"FIPS_DRBG_CHECK"},
 {ERR_FUNC(FIPS_F_FIPS_DRBG_CPRNG_TEST),	"FIPS_DRBG_CPRNG_TEST"},
@@ -95,6 +99,8 @@ static ERR_STRING_DATA FIPS_str_functs[]=
 {ERR_FUNC(FIPS_F_FIPS_DRBG_NEW),	"FIPS_drbg_new"},
 {ERR_FUNC(FIPS_F_FIPS_DRBG_RESEED),	"FIPS_drbg_reseed"},
 {ERR_FUNC(FIPS_F_FIPS_DRBG_SINGLE_KAT),	"FIPS_DRBG_SINGLE_KAT"},
+{ERR_FUNC(FIPS_F_FIPS_DSA_SIGN_DIGEST),	"FIPS_dsa_sign_digest"},
+{ERR_FUNC(FIPS_F_FIPS_DSA_VERIFY_DIGEST),	"FIPS_dsa_verify_digest"},
 {ERR_FUNC(FIPS_F_FIPS_GET_ENTROPY),	"FIPS_GET_ENTROPY"},
 {ERR_FUNC(FIPS_F_FIPS_MODE_SET),	"FIPS_mode_set"},
 {ERR_FUNC(FIPS_F_FIPS_PKEY_SIGNATURE_TEST),	"fips_pkey_signature_test"},
@@ -104,6 +110,8 @@ static ERR_STRING_DATA FIPS_str_functs[]=
 {ERR_FUNC(FIPS_F_FIPS_RAND_SEED),	"FIPS_rand_seed"},
 {ERR_FUNC(FIPS_F_FIPS_RAND_SET_METHOD),	"FIPS_rand_set_method"},
 {ERR_FUNC(FIPS_F_FIPS_RAND_STATUS),	"FIPS_rand_status"},
+{ERR_FUNC(FIPS_F_FIPS_RSA_SIGN_DIGEST),	"FIPS_rsa_sign_digest"},
+{ERR_FUNC(FIPS_F_FIPS_RSA_VERIFY_DIGEST),	"FIPS_rsa_verify_digest"},
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_AES),	"FIPS_selftest_aes"},
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_AES_CCM),	"FIPS_selftest_aes_ccm"},
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_AES_GCM),	"FIPS_selftest_aes_gcm"},
@@ -115,6 +123,7 @@ static ERR_STRING_DATA FIPS_str_functs[]=
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_HMAC),	"FIPS_selftest_hmac"},
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_SHA1),	"FIPS_selftest_sha1"},
 {ERR_FUNC(FIPS_F_FIPS_SELFTEST_X931),	"FIPS_selftest_x931"},
+{ERR_FUNC(FIPS_F_FIPS_SET_PRNG_KEY),	"FIPS_SET_PRNG_KEY"},
 {ERR_FUNC(FIPS_F_HASH_FINAL),	"HASH_FINAL"},
 {ERR_FUNC(FIPS_F_RSA_BUILTIN_KEYGEN),	"RSA_BUILTIN_KEYGEN"},
 {ERR_FUNC(FIPS_F_RSA_EAY_INIT),	"RSA_EAY_INIT"},
diff --git a/fips/dsa/fips_dsa_sign.c b/fips/dsa/fips_dsa_sign.c
index 16689309b9..ea1bd87303 100644
--- a/fips/dsa/fips_dsa_sign.c
+++ b/fips/dsa/fips_dsa_sign.c
@@ -3,7 +3,7 @@
  * project 2007.
  */
 /* ====================================================================
- * Copyright (c) 2007 The OpenSSL Project.  All rights reserved.
+ * Copyright (c) 2011 The OpenSSL Project.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -84,7 +84,11 @@ DSA_SIG * FIPS_dsa_sign_ctx(DSA *dsa, EVP_MD_CTX *ctx)
 
 DSA_SIG * FIPS_dsa_sign_digest(DSA *dsa, const unsigned char *dig, int dlen)
 	{
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_DSA_SIGN_DIGEST, FIPS_R_SELFTEST_FAILED);
+		return NULL;
+		}
 	return dsa->meth->dsa_do_sign(dig, dlen, dsa);
 	}
 
@@ -102,7 +106,11 @@ int FIPS_dsa_verify_ctx(DSA *dsa, EVP_MD_CTX *ctx, DSA_SIG *s)
 int FIPS_dsa_verify_digest(DSA *dsa,
 				const unsigned char *dig, int dlen, DSA_SIG *s)
 	{
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_DSA_VERIFY_DIGEST, FIPS_R_SELFTEST_FAILED);
+		return -1;
+		}
 	return dsa->meth->dsa_do_verify(dig,dlen,s,dsa);
 	}
 
diff --git a/fips/fips.h b/fips/fips.h
index e07a795525..c37c32b2fd 100644
--- a/fips/fips.h
+++ b/fips/fips.h
@@ -249,8 +249,12 @@ void ERR_load_FIPS_strings(void);
 #define FIPS_F_FIPS_CHECK_INCORE_FINGERPRINT		 107
 #define FIPS_F_FIPS_CHECK_RSA				 108
 #define FIPS_F_FIPS_CHECK_RSA_PRNG			 150
+#define FIPS_F_FIPS_CIPHER				 160
 #define FIPS_F_FIPS_CIPHERINIT				 109
+#define FIPS_F_FIPS_CIPHER_CTX_CTRL			 161
+#define FIPS_F_FIPS_DIGESTFINAL				 158
 #define FIPS_F_FIPS_DIGESTINIT				 110
+#define FIPS_F_FIPS_DIGESTUPDATE			 159
 #define FIPS_F_FIPS_DRBG_BYTES				 111
 #define FIPS_F_FIPS_DRBG_CHECK				 146
 #define FIPS_F_FIPS_DRBG_CPRNG_TEST			 112
@@ -261,6 +265,8 @@ void ERR_load_FIPS_strings(void);
 #define FIPS_F_FIPS_DRBG_NEW				 117
 #define FIPS_F_FIPS_DRBG_RESEED				 118
 #define FIPS_F_FIPS_DRBG_SINGLE_KAT			 119
+#define FIPS_F_FIPS_DSA_SIGN_DIGEST			 154
+#define FIPS_F_FIPS_DSA_VERIFY_DIGEST			 155
 #define FIPS_F_FIPS_GET_ENTROPY				 147
 #define FIPS_F_FIPS_MODE_SET				 120
 #define FIPS_F_FIPS_PKEY_SIGNATURE_TEST			 121
@@ -270,6 +276,8 @@ void ERR_load_FIPS_strings(void);
 #define FIPS_F_FIPS_RAND_SEED				 125
 #define FIPS_F_FIPS_RAND_SET_METHOD			 126
 #define FIPS_F_FIPS_RAND_STATUS				 127
+#define FIPS_F_FIPS_RSA_SIGN_DIGEST			 156
+#define FIPS_F_FIPS_RSA_VERIFY_DIGEST			 157
 #define FIPS_F_FIPS_SELFTEST_AES			 128
 #define FIPS_F_FIPS_SELFTEST_AES_CCM			 145
 #define FIPS_F_FIPS_SELFTEST_AES_GCM			 129
@@ -281,6 +289,7 @@ void ERR_load_FIPS_strings(void);
 #define FIPS_F_FIPS_SELFTEST_HMAC			 134
 #define FIPS_F_FIPS_SELFTEST_SHA1			 135
 #define FIPS_F_FIPS_SELFTEST_X931			 136
+#define FIPS_F_FIPS_SET_PRNG_KEY			 153
 #define FIPS_F_HASH_FINAL				 137
 #define FIPS_F_RSA_BUILTIN_KEYGEN			 138
 #define FIPS_F_RSA_EAY_INIT				 149
diff --git a/fips/rand/fips_rand.c b/fips/rand/fips_rand.c
index f8de94291d..cb9184e1f7 100644
--- a/fips/rand/fips_rand.c
+++ b/fips/rand/fips_rand.c
@@ -136,7 +136,11 @@ static void fips_rand_prng_reset(FIPS_PRNG_CTX *ctx)
 static int fips_set_prng_key(FIPS_PRNG_CTX *ctx,
 			const unsigned char *key, unsigned int keylen)
 	{
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_SET_PRNG_KEY, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
 	if (keylen != 16 && keylen != 24 && keylen != 32)
 		{
 		/* error: invalid key size */
diff --git a/fips/rsa/fips_rsa_sign.c b/fips/rsa/fips_rsa_sign.c
index 46d0d4061a..c68c00787d 100644
--- a/fips/rsa/fips_rsa_sign.c
+++ b/fips/rsa/fips_rsa_sign.c
@@ -219,7 +219,11 @@ int FIPS_rsa_sign_digest(RSA *rsa, const unsigned char *md, int md_len,
 	/* Largest DigestInfo: 19 (max encoding) + max MD */
 	unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
 
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_RSA_SIGN_DIGEST, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
 
 	md_type = M_EVP_MD_type(mhash);
 
@@ -322,14 +326,18 @@ int FIPS_rsa_verify_digest(RSA *rsa, const unsigned char *dig, int diglen,
 	int md_type;
 	int rsa_dec_pad_mode;
 
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_RSA_VERIFY_DIGEST, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
+
 	if (siglen != (unsigned int)RSA_size(rsa))
 		{
 		RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_WRONG_SIGNATURE_LENGTH);
 		return(0);
 		}
 
-	FIPS_selftest_check();
-
 	md_type = M_EVP_MD_type(mhash);
 
 	s= OPENSSL_malloc((unsigned int)siglen);
diff --git a/fips/utl/fips_enc.c b/fips/utl/fips_enc.c
index a25e5a1e5c..55a880d06e 100644
--- a/fips/utl/fips_enc.c
+++ b/fips/utl/fips_enc.c
@@ -256,11 +256,15 @@ int FIPS_cipher_ctx_cleanup(EVP_CIPHER_CTX *c)
 int FIPS_cipher_ctx_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr)
 {
 	int ret;
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_CIPHER_CTX_CTRL, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
 	if(!ctx->cipher) {
 		EVPerr(EVP_F_FIPS_CIPHER_CTX_CTRL, EVP_R_NO_CIPHER_SET);
 		return 0;
 	}
-	FIPS_selftest_check();
 
 	if(!ctx->cipher->ctrl) {
 		EVPerr(EVP_F_FIPS_CIPHER_CTX_CTRL, EVP_R_CTRL_NOT_IMPLEMENTED);
@@ -327,6 +331,10 @@ int FIPS_cipher_ctx_set_key_length(EVP_CIPHER_CTX *ctx, int keylen)
 int FIPS_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			const unsigned char *in, unsigned int inl)
 	{
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_CIPHER, FIPS_R_SELFTEST_FAILED);
+		return -1;
+		}
 	return ctx->cipher->do_cipher(ctx,out,in,inl);
 	}
diff --git a/fips/utl/fips_md.c b/fips/utl/fips_md.c
index 37149506f3..556267994c 100644
--- a/fips/utl/fips_md.c
+++ b/fips/utl/fips_md.c
@@ -204,7 +204,11 @@ int FIPS_digestinit(EVP_MD_CTX *ctx, const EVP_MD *type)
 
 int FIPS_digestupdate(EVP_MD_CTX *ctx, const void *data, size_t count)
 	{
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_DIGESTUPDATE, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
 	return ctx->update(ctx,data,count);
 	}
 
@@ -213,7 +217,11 @@ int FIPS_digestfinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *size)
 	{
 	int ret;
 
-	FIPS_selftest_check();
+	if (FIPS_selftest_failed())
+		{
+		FIPSerr(FIPS_F_FIPS_DIGESTFINAL, FIPS_R_SELFTEST_FAILED);
+		return 0;
+		}
 
 	OPENSSL_assert(ctx->digest->md_size <= EVP_MAX_MD_SIZE);
 	ret=ctx->digest->final(ctx,md);