Separate fips and non fips code for key operations

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12745)

7 files changed, 347 insertions, 128 deletions

diff --git a/providers/common/build.info b/providers/common/build.info
index fb04883507..1e0ea4e5dd 100644
--- a/providers/common/build.info
+++ b/providers/common/build.info
@@ -1,6 +1,6 @@
 SUBDIRS=der
 
 SOURCE[../libcommon.a]=provider_err.c provider_ctx.c
-$FIPSCOMMON=provider_util.c capabilities.c bio_prov.c
-SOURCE[../libnonfips.a]=$FIPSCOMMON nid_to_name.c
-SOURCE[../libfips.a]=$FIPSCOMMON
+$FIPSCOMMON=provider_util.c capabilities.c bio_prov.c digest_to_nid.c
+SOURCE[../libnonfips.a]=$FIPSCOMMON nid_to_name.c check_default.c
+SOURCE[../libfips.a]=$FIPSCOMMON check_fips.c
diff --git a/providers/common/check_default.c b/providers/common/check_default.c
new file mode 100644
index 0000000000..2bfa690573
--- /dev/null
+++ b/providers/common/check_default.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "internal/deprecated.h"
+
+#include <openssl/rsa.h>
+#include <openssl/core.h>
+#include <openssl/core_names.h>
+#include <openssl/obj_mac.h>
+#include "prov/check.h"
+#include "internal/nelem.h"
+
+int rsa_check_key(ossl_unused const RSA *rsa, ossl_unused int protect)
+{
+    return 1;
+}
+
+#ifndef OPENSSL_NO_EC
+int ec_check_key(ossl_unused const EC_KEY *ec, ossl_unused int protect)
+{
+    return 1;
+}
+#endif
+
+#ifndef OPENSSL_NO_DSA
+int dsa_check_key(ossl_unused const DSA *dsa, ossl_unused int sign)
+{
+    return 1;
+}
+#endif
+
+#ifndef OPENSSL_NO_DH
+int dh_check_key(const DH *dh)
+{
+    return 1;
+}
+#endif
+
+int digest_is_allowed(ossl_unused const EVP_MD *md)
+{
+    return 1;
+}
+
+int digest_get_approved_nid_with_sha1(const EVP_MD *md,
+                                      ossl_unused int sha1_allowed)
+{
+    return digest_get_approved_nid(md);
+}
+
+int digest_rsa_sign_get_md_nid(const EVP_MD *md, ossl_unused int sha1_allowed)
+{
+    int mdnid;
+
+    static const OSSL_ITEM name_to_nid[] = {
+        { NID_md5,       OSSL_DIGEST_NAME_MD5       },
+        { NID_md5_sha1,  OSSL_DIGEST_NAME_MD5_SHA1  },
+        { NID_md2,       OSSL_DIGEST_NAME_MD2       },
+        { NID_md4,       OSSL_DIGEST_NAME_MD4       },
+        { NID_mdc2,      OSSL_DIGEST_NAME_MDC2      },
+        { NID_ripemd160, OSSL_DIGEST_NAME_RIPEMD160 },
+    };
+    if (md == NULL)
+        return NID_undef;
+
+    mdnid = digest_get_approved_nid_with_sha1(md, 1);
+    if (mdnid == NID_undef)
+        mdnid = digest_md_to_nid(md, name_to_nid, OSSL_NELEM(name_to_nid));
+    return mdnid;
+}
diff --git a/providers/common/check_fips.c b/providers/common/check_fips.c
new file mode 100644
index 0000000000..1ce0718e1d
--- /dev/null
+++ b/providers/common/check_fips.c
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "internal/deprecated.h"
+
+#include <openssl/rsa.h>
+#include <openssl/dsa.h>
+#include <openssl/dh.h>
+#include <openssl/ec.h>
+#include <openssl/err.h>
+#include <openssl/core_names.h>
+#include <openssl/obj_mac.h>
+#include "prov/check.h"
+#include "prov/providercommonerr.h"
+
+/*
+ * FIPS requires a minimum security strength of 112 bits (for encryption or
+ * signing), and for legacy purposes 80 bits (for decryption or verifying).
+ * Set protect = 1 for encryption or signing operations, or 0 otherwise.
+ */
+int rsa_check_key(const RSA *rsa, int protect)
+{
+    int sz = RSA_bits(rsa);
+
+    return protect ? (sz >= 2048) : (sz >= 1024);
+}
+
+#ifndef OPENSSL_NO_EC
+/*
+ * In FIPS mode:
+ * protect should be 1 for any operations that need 112 bits of security
+ * strength (such as signing, and key exchange), or 0 for operations that allow
+ * a lower security strength (such as verify).
+ *
+ * For ECDH key agreement refer to SP800-56A
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-56Ar3.pdf
+ * "Appendix D"
+ *
+ * For ECDSA signatures refer to
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar2.pdf
+ * "Table 2"
+ */
+int ec_check_key(const EC_KEY *ec, int protect)
+{
+    int nid, strength;
+    const char *curve_name;
+    const EC_GROUP *group = EC_KEY_get0_group(ec);
+
+    if (group == NULL) {
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE, "No group");
+        return 0;
+    }
+    nid = EC_GROUP_get_curve_name(group);
+    if (nid == NID_undef) {
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
+                       "Explicit curves are not allowed in fips mode");
+        return 0;
+    }
+
+    curve_name = EC_curve_nid2nist(nid);
+    if (curve_name == NULL) {
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
+                       "Curve %s is not approved in FIPS mode", curve_name);
+        return 0;
+    }
+
+    /*
+     * For EC the security strength is the (order_bits / 2)
+     * e.g. P-224 is 112 bits.
+     */
+    strength = EC_GROUP_order_bits(group) / 2;
+    /* The min security strength allowed for legacy verification is 80 bits */
+    if (strength < 80) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE);
+        return 0;
+    }
+
+    /*
+     * For signing or key agreement only allow curves with at least 112 bits of
+     * security strength
+     */
+    if (protect && strength < 112) {
+        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
+                       "Curve %s cannot be used for signing", curve_name);
+        return 0;
+    }
+    return 1;
+}
+#endif /* OPENSSL_NO_EC */
+
+#ifndef OPENSSL_NO_DSA
+/*
+ * Check for valid key sizes if fips mode. Refer to
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar2.pdf
+ * "Table 2"
+ */
+int dsa_check_key(const DSA *dsa, int sign)
+{
+    size_t L, N;
+    const BIGNUM *p, *q;
+
+    if (dsa == NULL)
+        return 0;
+
+    p = DSA_get0_p(dsa);
+    q = DSA_get0_q(dsa);
+    if (p == NULL || q == NULL)
+        return 0;
+
+    L = BN_num_bits(p);
+    N = BN_num_bits(q);
+
+    /*
+     * Valid sizes or verification - Note this could be a fips186-2 type
+     * key - so we allow 512 also. When this is no longer suppported the
+     * lower bound should be increased to 1024.
+     */
+    if (!sign)
+        return (L >= 512 && N >= 160);
+
+     /* Valid sizes for both sign and verify */
+    if (L == 2048 && (N == 224 || N == 256))
+        return 1;
+    return (L == 3072 && N == 256);
+}
+#endif /* OPENSSL_NO_DSA */
+
+#ifndef OPENSSL_NO_DH
+/*
+ * For DH key agreement refer to SP800-56A
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-56Ar3.pdf
+ * "Section 5.5.1.1FFC Domain Parameter Selection/Generation" and
+ * "Appendix D" FFC Safe-prime Groups
+ */
+int dh_check_key(const DH *dh)
+{
+    size_t L, N;
+    const BIGNUM *p, *q;
+
+    if (dh == NULL)
+        return 0;
+
+    p = DH_get0_p(dh);
+    q = DH_get0_q(dh);
+    if (p == NULL || q == NULL)
+        return 0;
+
+    L = BN_num_bits(p);
+    if (L < 2048)
+        return 0;
+
+    /* If it is a safe prime group then it is ok */
+    if (DH_get_nid(dh))
+        return 1;
+
+    /* If not then it must be FFC, which only allows certain sizes. */
+    N = BN_num_bits(q);
+
+    return (L == 2048 && (N == 224 || N == 256));
+}
+#endif /* OPENSSL_NO_DH */
+
+int digest_get_approved_nid_with_sha1(const EVP_MD *md, int sha1_allowed)
+{
+    int mdnid = digest_get_approved_nid(md);
+
+    if (mdnid == NID_sha1 && !sha1_allowed)
+        mdnid = NID_undef;
+
+    return mdnid;
+}
+
+int digest_is_allowed(const EVP_MD *md)
+{
+    return (digest_get_approved_nid(md) != NID_undef);
+}
+
+int digest_rsa_sign_get_md_nid(const EVP_MD *md, int sha1_allowed)
+{
+    return digest_get_approved_nid_with_sha1(md, sha1_allowed);
+}
diff --git a/providers/common/digest_to_nid.c b/providers/common/digest_to_nid.c
new file mode 100644
index 0000000000..4f59709f5d
--- /dev/null
+++ b/providers/common/digest_to_nid.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <openssl/objects.h>
+#include <openssl/core_names.h>
+#include <openssl/evp.h>
+#include <openssl/core.h>
+#include "prov/check.h"
+#include "internal/nelem.h"
+
+/*
+ * Internal library code deals with NIDs, so we need to translate from a name.
+ * We do so using EVP_MD_is_a(), and therefore need a name to NID map.
+ */
+int digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it, size_t it_len)
+{
+    size_t i;
+    int mdnid = NID_undef;
+
+    if (md == NULL)
+        goto end;
+
+    for (i = 0; i < it_len; i++) {
+        if (EVP_MD_is_a(md, it[i].ptr)) {
+            mdnid = (int)it[i].id;
+            break;
+        }
+    }
+ end:
+    return mdnid;
+}
+
+/*
+ * Retrieve one of the FIPs approved hash algorithms by nid.
+ * See FIPS 180-4 "Secure Hash Standard" and FIPS 202 - SHA-3.
+ */
+int digest_get_approved_nid(const EVP_MD *md)
+{
+    static const OSSL_ITEM name_to_nid[] = {
+        { NID_sha1,      OSSL_DIGEST_NAME_SHA1      },
+        { NID_sha224,    OSSL_DIGEST_NAME_SHA2_224  },
+        { NID_sha256,    OSSL_DIGEST_NAME_SHA2_256  },
+        { NID_sha384,    OSSL_DIGEST_NAME_SHA2_384  },
+        { NID_sha512,    OSSL_DIGEST_NAME_SHA2_512  },
+        { NID_sha512_224, OSSL_DIGEST_NAME_SHA2_512_224 },
+        { NID_sha512_256, OSSL_DIGEST_NAME_SHA2_512_256 },
+        { NID_sha3_224,  OSSL_DIGEST_NAME_SHA3_224  },
+        { NID_sha3_256,  OSSL_DIGEST_NAME_SHA3_256  },
+        { NID_sha3_384,  OSSL_DIGEST_NAME_SHA3_384  },
+        { NID_sha3_512,  OSSL_DIGEST_NAME_SHA3_512  },
+    };
+
+    return digest_md_to_nid(md, name_to_nid, OSSL_NELEM(name_to_nid));
+}
diff --git a/providers/common/include/prov/check.h b/providers/common/include/prov/check.h
new file mode 100644
index 0000000000..30425840b5
--- /dev/null
+++ b/providers/common/include/prov/check.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/* Functions that have different implementations for the FIPS_MODULE */
+int rsa_check_key(const RSA *rsa, int protect);
+int ec_check_key(const EC_KEY *ec, int protect);
+int dsa_check_key(const DSA *dsa, int sign);
+int dh_check_key(const DH *dh);
+
+int digest_is_allowed(const EVP_MD *md);
+int digest_get_approved_nid_with_sha1(const EVP_MD *md, int sha1_allowed);
+int digest_rsa_sign_get_md_nid(const EVP_MD *md, ossl_unused int sha1_allowed);
+
+/* Functions that are common */
+int digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it, size_t it_len);
+int digest_get_approved_nid(const EVP_MD *md);
diff --git a/providers/common/include/prov/provider_util.h b/providers/common/include/prov/provider_util.h
index d4fbd9b74b..d964f832ad 100644
--- a/providers/common/include/prov/provider_util.h
+++ b/providers/common/include/prov/provider_util.h
@@ -128,8 +128,3 @@ typedef struct ag_capable_st {
  */
 void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in,
                                          OSSL_ALGORITHM *out);
-
-int ossl_prov_digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it,
-                               size_t it_len);
-int ossl_prov_digest_get_approved_nid(const EVP_MD *md, int sha1_allowed);
-int ossl_prov_ec_check(const EC_KEY *ec, int protect);
diff --git a/providers/common/provider_util.c b/providers/common/provider_util.c
index f27171a830..4259d7167a 100644
--- a/providers/common/provider_util.c
+++ b/providers/common/provider_util.c
@@ -296,123 +296,3 @@ void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in,
         out[j++] = in[i].alg;
     }
 }
-
-/*
- * Internal library code deals with NIDs, so we need to translate from a name.
- * We do so using EVP_MD_is_a(), and therefore need a name to NID map.
- */
-int ossl_prov_digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it,
-                               size_t it_len)
-{
-    size_t i;
-    int mdnid = NID_undef;
-
-    if (md == NULL)
-        goto end;
-
-    for (i = 0; i < it_len; i++) {
-        if (EVP_MD_is_a(md, it[i].ptr)) {
-            mdnid = (int)it[i].id;
-            break;
-        }
-    }
- end:
-    return mdnid;
-}
-
-/*
- * Retrieve one of the FIPs approved hash algorithms by nid.
- * See FIPS 180-4 "Secure Hash Standard" and
- * FIPS 202 - SHA-3.
- *
- * NOTE: For some operations SHA1 is not allowed. This check is only enabled
- * for the FIPS_MODULE.
- */
-int ossl_prov_digest_get_approved_nid(const EVP_MD *md, int sha1_allowed)
-{
-    int mdnid;
-
-    static const OSSL_ITEM name_to_nid[] = {
-        { NID_sha1,      OSSL_DIGEST_NAME_SHA1      },
-        { NID_sha224,    OSSL_DIGEST_NAME_SHA2_224  },
-        { NID_sha256,    OSSL_DIGEST_NAME_SHA2_256  },
-        { NID_sha384,    OSSL_DIGEST_NAME_SHA2_384  },
-        { NID_sha512,    OSSL_DIGEST_NAME_SHA2_512  },
-        { NID_sha512_224, OSSL_DIGEST_NAME_SHA2_512_224 },
-        { NID_sha512_256, OSSL_DIGEST_NAME_SHA2_512_256 },
-        { NID_sha3_224,  OSSL_DIGEST_NAME_SHA3_224  },
-        { NID_sha3_256,  OSSL_DIGEST_NAME_SHA3_256  },
-        { NID_sha3_384,  OSSL_DIGEST_NAME_SHA3_384  },
-        { NID_sha3_512,  OSSL_DIGEST_NAME_SHA3_512  },
-    };
-
-    mdnid = ossl_prov_digest_md_to_nid(md, name_to_nid, OSSL_NELEM(name_to_nid));
-#ifdef FIPS_MODULE
-    if (mdnid == NID_sha1 && !sha1_allowed)
-        mdnid = NID_undef;
-#endif
-    return mdnid;
-}
-
-/*
- * In FIPS mode:
- * protect should be 1 for any operations that need 112 bits of security
- * strength (such as signing, and key exchange), or 0 for operations that allow
- * a lower security strength (such as verify).
- *
- * For ECDH key agreement refer to SP800-56A
- * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-56Ar3.pdf
- * "Appendix D"
- *
- * For ECDSA signatures refer to
- * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar2.pdf
- * "Table 2"
- */
-int ossl_prov_ec_check(const EC_KEY *ec, int protect)
-{
-#ifdef FIPS_MODULE
-    int nid, strength;
-    const char *curve_name;
-    const EC_GROUP *group = EC_KEY_get0_group(ec);
-
-    if (group == NULL) {
-        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE, "No group");
-        return 0;
-    }
-    nid = EC_GROUP_get_curve_name(group);
-    if (nid == NID_undef) {
-        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
-                       "Explicit curves are not allowed in fips mode");
-        return 0;
-    }
-
-    curve_name = EC_curve_nid2nist(nid);
-    if (curve_name == NULL) {
-        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
-                       "Curve %s is not approved in FIPS mode", curve_name);
-        return 0;
-    }
-
-    /*
-     * For EC the security strength is the (order_bits / 2)
-     * e.g. P-224 is 112 bits.
-     */
-    strength = EC_GROUP_order_bits(group) / 2;
-    /* The min security strength allowed for legacy verification is 80 bits */
-    if (strength < 80) {
-        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE);
-        return 0;
-    }
-
-    /*
-     * For signing/or key agreement only allow curves with at least 112 bits of
-     * security strength
-     */
-    if (protect && strength < 112) {
-        ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_CURVE,
-                       "Curve %s cannot be used for signing", curve_name);
-        return 0;
-    }
-#endif
-    return 1;
-}