Add fips checks for rsa signatures.

In fips mode SHA1 should not be allowed for signing, but may be present for verifying.
Add keysize check.
Add missing 'ossl_unused' to gettable and settable methods.
Update fips related tests that have these restrictions.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12745)

3 files changed, 130 insertions, 44 deletions

diff --git a/providers/common/include/prov/provider_util.h b/providers/common/include/prov/provider_util.h
index d964f832ad..7306e6aa8c 100644
--- a/providers/common/include/prov/provider_util.h
+++ b/providers/common/include/prov/provider_util.h
@@ -128,3 +128,7 @@ typedef struct ag_capable_st {
  */
 void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in,
                                          OSSL_ALGORITHM *out);
+
+int ossl_prov_digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it,
+                               size_t it_len);
+int ossl_prov_digest_get_approved_nid(const EVP_MD *md, int sha1_allowed);
diff --git a/providers/common/provider_util.c b/providers/common/provider_util.c
index 1bd514221f..51ade22a37 100644
--- a/providers/common/provider_util.c
+++ b/providers/common/provider_util.c
@@ -14,6 +14,8 @@
 #include <openssl/core_names.h>
 #include <openssl/err.h>
 #include "prov/provider_util.h"
+#include "prov/providercommonerr.h"
+#include "internal/nelem.h"
 
 void ossl_prov_cipher_reset(PROV_CIPHER *pc)
 {
@@ -294,3 +296,60 @@ void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in,
         out[j++] = in[i].alg;
     }
 }
+
+/*
+ * Internal library code deals with NIDs, so we need to translate from a name.
+ * We do so using EVP_MD_is_a(), and therefore need a name to NID map.
+ */
+int ossl_prov_digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it,
+                               size_t it_len)
+{
+    size_t i;
+    int mdnid = NID_undef;
+
+    if (md == NULL)
+        goto end;
+
+    for (i = 0; i < it_len; i++) {
+        if (EVP_MD_is_a(md, it[i].ptr)) {
+            mdnid = (int)it[i].id;
+            break;
+        }
+    }
+ end:
+    return mdnid;
+}
+
+/*
+ * Retrieve one of the FIPs approved hash algorithms by nid.
+ * See FIPS 180-4 "Secure Hash Standard" and
+ * FIPS 202 - SHA-3.
+ *
+ * NOTE: For some operations SHA1 is not allowed. This check is only enabled
+ * for the FIPS_MODULE.
+ */
+int ossl_prov_digest_get_approved_nid(const EVP_MD *md, int sha1_allowed)
+{
+    int mdnid;
+
+    static const OSSL_ITEM name_to_nid[] = {
+        { NID_sha1,      OSSL_DIGEST_NAME_SHA1      },
+        { NID_sha224,    OSSL_DIGEST_NAME_SHA2_224  },
+        { NID_sha256,    OSSL_DIGEST_NAME_SHA2_256  },
+        { NID_sha384,    OSSL_DIGEST_NAME_SHA2_384  },
+        { NID_sha512,    OSSL_DIGEST_NAME_SHA2_512  },
+        { NID_sha512_224, OSSL_DIGEST_NAME_SHA2_512_224 },
+        { NID_sha512_256, OSSL_DIGEST_NAME_SHA2_512_256 },
+        { NID_sha3_224,  OSSL_DIGEST_NAME_SHA3_224  },
+        { NID_sha3_256,  OSSL_DIGEST_NAME_SHA3_256  },
+        { NID_sha3_384,  OSSL_DIGEST_NAME_SHA3_384  },
+        { NID_sha3_512,  OSSL_DIGEST_NAME_SHA3_512  },
+    };
+
+    mdnid = ossl_prov_digest_md_to_nid(md, name_to_nid, OSSL_NELEM(name_to_nid));
+#ifdef FIPS_MODULE
+    if (mdnid == NID_sha1 && !sha1_allowed)
+        mdnid = NID_undef;
+#endif
+    return mdnid;
+}
diff --git a/providers/implementations/signature/rsa.c b/providers/implementations/signature/rsa.c
index 678d3e7b66..c7f3f6f6cd 100644
--- a/providers/implementations/signature/rsa.c
+++ b/providers/implementations/signature/rsa.c
@@ -30,6 +30,9 @@
 #include "prov/implementations.h"
 #include "prov/provider_ctx.h"
 #include "prov/der_rsa.h"
+#include "prov/provider_util.h"
+
+#define RSA_DEFAULT_DIGEST_NAME OSSL_DIGEST_NAME_SHA1
 
 static OSSL_FUNC_signature_newctx_fn rsa_newctx;
 static OSSL_FUNC_signature_sign_init_fn rsa_sign_init;
@@ -84,7 +87,7 @@ typedef struct {
      */
     unsigned int flag_allow_md : 1;
 
-    /* The Algorithm Identifier of the combined signature agorithm */
+    /* The Algorithm Identifier of the combined signature algorithm */
     unsigned char aid_buf[128];
     unsigned char *aid;
     size_t  aid_len;
@@ -117,49 +120,58 @@ static size_t rsa_get_md_size(const PROV_RSA_CTX *prsactx)
     return 0;
 }
 
-static int rsa_get_md_nid(const EVP_MD *md)
+static int rsa_get_md_nid_check(const PROV_RSA_CTX *ctx, const EVP_MD *md,
+                                int sha1_allowed)
 {
-    /*
-     * Because the RSA library deals with NIDs, we need to translate.
-     * We do so using EVP_MD_is_a(), and therefore need a name to NID
-     * map.
-     */
+    int mdnid = NID_undef;
+
+    #ifndef FIPS_MODULE
     static const OSSL_ITEM name_to_nid[] = {
-        { NID_sha1,      OSSL_DIGEST_NAME_SHA1      },
-        { NID_sha224,    OSSL_DIGEST_NAME_SHA2_224  },
-        { NID_sha256,    OSSL_DIGEST_NAME_SHA2_256  },
-        { NID_sha384,    OSSL_DIGEST_NAME_SHA2_384  },
-        { NID_sha512,    OSSL_DIGEST_NAME_SHA2_512  },
-        { NID_sha512_224, OSSL_DIGEST_NAME_SHA2_512_224 },
-        { NID_sha512_256, OSSL_DIGEST_NAME_SHA2_512_256 },
         { NID_md5,       OSSL_DIGEST_NAME_MD5       },
         { NID_md5_sha1,  OSSL_DIGEST_NAME_MD5_SHA1  },
         { NID_md2,       OSSL_DIGEST_NAME_MD2       },
         { NID_md4,       OSSL_DIGEST_NAME_MD4       },
         { NID_mdc2,      OSSL_DIGEST_NAME_MDC2      },
         { NID_ripemd160, OSSL_DIGEST_NAME_RIPEMD160 },
-        { NID_sha3_224,  OSSL_DIGEST_NAME_SHA3_224  },
-        { NID_sha3_256,  OSSL_DIGEST_NAME_SHA3_256  },
-        { NID_sha3_384,  OSSL_DIGEST_NAME_SHA3_384  },
-        { NID_sha3_512,  OSSL_DIGEST_NAME_SHA3_512  },
     };
-    size_t i;
-    int mdnid = NID_undef;
+    #endif
 
     if (md == NULL)
         goto end;
 
-    for (i = 0; i < OSSL_NELEM(name_to_nid); i++) {
-        if (EVP_MD_is_a(md, name_to_nid[i].ptr)) {
-            mdnid = (int)name_to_nid[i].id;
-            break;
-        }
-    }
+    mdnid = ossl_prov_digest_get_approved_nid(md, sha1_allowed);
 
- end:
+    #ifndef FIPS_MODULE
+    if (mdnid == NID_undef)
+        mdnid = ossl_prov_digest_md_to_nid(md, name_to_nid,
+                                           OSSL_NELEM(name_to_nid));
+    #endif
+    end:
     return mdnid;
 }
 
+static int rsa_get_md_nid(const PROV_RSA_CTX *ctx, const EVP_MD *md)
+{
+    return rsa_get_md_nid_check(ctx, md, ctx->operation != EVP_PKEY_OP_SIGN);
+}
+
+static int rsa_get_md_mgf1_nid(const PROV_RSA_CTX *ctx, const EVP_MD *md)
+{
+    /* The default for mgf1 is SHA1 - so allow this */
+    return rsa_get_md_nid_check(ctx, md, 1);
+}
+
+static int rsa_check_key_size(const PROV_RSA_CTX *prsactx)
+{
+#ifdef FIPS_MODULE
+    int sz = RSA_bits(prsactx->rsa);
+
+    return (prsactx->operation == EVP_PKEY_OP_SIGN) ? (sz >= 2048) : (sz >= 1024);
+#else
+    return 1;
+#endif
+}
+
 static int rsa_check_padding(int mdnid, int padding)
 {
     if (padding == RSA_NO_PADDING) {
@@ -226,9 +238,9 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
         mdprops = ctx->propq;
 
     if (mdname != NULL) {
-        EVP_MD *md = EVP_MD_fetch(ctx->libctx, mdname, mdprops);
-        int md_nid = rsa_get_md_nid(md);
         WPACKET pkt;
+        EVP_MD *md = EVP_MD_fetch(ctx->libctx, mdname, mdprops);
+        int md_nid = rsa_get_md_nid(ctx, md);
         size_t mdname_len = strlen(mdname);
 
         if (md == NULL
@@ -281,6 +293,7 @@ static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
                              const char *mdprops)
 {
     size_t len;
+    EVP_MD *md = NULL;
 
     if (mdprops == NULL)
         mdprops = ctx->propq;
@@ -288,11 +301,18 @@ static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
     if (ctx->mgf1_mdname[0] != '\0')
         EVP_MD_free(ctx->mgf1_md);
 
-    if ((ctx->mgf1_md = EVP_MD_fetch(ctx->libctx, mdname, mdprops)) == NULL) {
+    if ((md = EVP_MD_fetch(ctx->libctx, mdname, mdprops)) == NULL) {
         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
                        "%s could not be fetched", mdname);
         return 0;
     }
+    if (rsa_get_md_mgf1_nid(ctx, md) == NID_undef) {
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
+                       "digest=%s", mdname);
+        EVP_MD_free(md);
+        return 0;
+    }
+    ctx->mgf1_md = md;
     len = OPENSSL_strlcpy(ctx->mgf1_mdname, mdname, sizeof(ctx->mgf1_mdname));
     if (len >= sizeof(ctx->mgf1_mdname)) {
         ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
@@ -303,7 +323,7 @@ static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
     return 1;
 }
 
-static int rsa_signature_init(void *vprsactx, void *vrsa, int operation)
+static int rsa_signverify_init(void *vprsactx, void *vrsa, int operation)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 
@@ -317,6 +337,11 @@ static int rsa_signature_init(void *vprsactx, void *vrsa, int operation)
     prsactx->rsa = vrsa;
     prsactx->operation = operation;
 
+    if (!rsa_check_key_size(prsactx)) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH);
+        return 0;
+    }
+
     /* Maximum for sign, auto for verify */
     prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
     prsactx->min_saltlen = -1;
@@ -413,7 +438,7 @@ static int rsa_sign_init(void *vprsactx, void *vrsa)
 {
     if (!ossl_prov_is_running())
         return 0;
-    return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_SIGN);
+    return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_SIGN);
 }
 
 static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen,
@@ -566,7 +591,7 @@ static int rsa_verify_recover_init(void *vprsactx, void *vrsa)
 {
     if (!ossl_prov_is_running())
         return 0;
-    return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFYRECOVER);
+    return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFYRECOVER);
 }
 
 static int rsa_verify_recover(void *vprsactx,
@@ -657,7 +682,7 @@ static int rsa_verify_init(void *vprsactx, void *vrsa)
 {
     if (!ossl_prov_is_running())
         return 0;
-    return rsa_signature_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFY);
+    return rsa_signverify_init(vprsactx, vrsa, EVP_PKEY_OP_VERIFY);
 }
 
 static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
@@ -751,7 +776,7 @@ static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
 
     if (prsactx != NULL)
         prsactx->flag_allow_md = 0;
-    if (!rsa_signature_init(vprsactx, vrsa, operation)
+    if (!rsa_signverify_init(vprsactx, vrsa, operation)
         || !rsa_setup_md(prsactx, mdname, NULL)) /* TODO RL */
         return 0;
 
@@ -813,9 +838,8 @@ static int rsa_digest_sign_final(void *vprsactx, unsigned char *sig,
      */
     if (sig != NULL) {
         /*
-         * TODO(3.0): There is the possibility that some externally provided
-         * digests exceed EVP_MAX_MD_SIZE. We should probably handle that somehow -
-         * but that problem is much larger than just in RSA.
+         * The digests used here are all known (see rsa_get_md_nid()), so they
+         * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
          */
         if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
             return 0;
@@ -850,9 +874,8 @@ int rsa_digest_verify_final(void *vprsactx, const unsigned char *sig,
         return 0;
 
     /*
-     * TODO(3.0): There is the possibility that some externally provided
-     * digests exceed EVP_MAX_MD_SIZE. We should probably handle that somehow -
-     * but that problem is much larger than just in RSA.
+     * The digests used here are all known (see rsa_get_md_nid()), so they
+     * should not exceed the internal buffer size of EVP_MAX_MD_SIZE.
      */
     if (!EVP_DigestFinal_ex(prsactx->mdctx, digest, &dlen))
         return 0;
@@ -1021,7 +1044,7 @@ static const OSSL_PARAM known_gettable_ctx_params[] = {
     OSSL_PARAM_END
 };
 
-static const OSSL_PARAM *rsa_gettable_ctx_params(ossl_unused void *provctx)
+static const OSSL_PARAM *rsa_gettable_ctx_params(ossl_unused void *vctx)
 {
     return known_gettable_ctx_params;
 }
@@ -1112,7 +1135,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
                 goto bad_pad;
             }
             if (prsactx->md == NULL
-                && !rsa_setup_md(prsactx, OSSL_DIGEST_NAME_SHA1, NULL)) {
+                && !rsa_setup_md(prsactx, RSA_DEFAULT_DIGEST_NAME, NULL)) {
                 return 0;
             }
             break;
@@ -1271,7 +1294,7 @@ static const OSSL_PARAM known_settable_ctx_params[] = {
     OSSL_PARAM_END
 };
 
-static const OSSL_PARAM *rsa_settable_ctx_params(ossl_unused void *provctx)
+static const OSSL_PARAM *rsa_settable_ctx_params(void *provctx)
 {
     /*
      * TODO(3.0): Should this function return a different set of settable ctx