Check DSA parameters for excessive sizes before validating

This avoids overly long computation of various validation
checks.

Fixes CVE-2024-4603

Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/24346)

1 files changed, 40 insertions, 4 deletions

diff --git a/crypto/dsa/dsa_check.c b/crypto/dsa/dsa_check.c
index 7b6d7df88f..e1375dfad9 100644
--- a/crypto/dsa/dsa_check.c
+++ b/crypto/dsa/dsa_check.c
@@ -19,8 +19,34 @@
 #include "dsa_local.h"
 #include "crypto/dsa.h"
 
+static int dsa_precheck_params(const DSA *dsa, int *ret)
+{
+    if (dsa->params.p == NULL || dsa->params.q == NULL) {
+        ERR_raise(ERR_LIB_DSA, DSA_R_BAD_FFC_PARAMETERS);
+        *ret = FFC_CHECK_INVALID_PQ;
+        return 0;
+    }
+
+    if (BN_num_bits(dsa->params.p) > OPENSSL_DSA_MAX_MODULUS_BITS) {
+        ERR_raise(ERR_LIB_DSA, DSA_R_MODULUS_TOO_LARGE);
+        *ret = FFC_CHECK_INVALID_PQ;
+        return 0;
+    }
+
+    if (BN_num_bits(dsa->params.q) >= BN_num_bits(dsa->params.p)) {
+        ERR_raise(ERR_LIB_DSA, DSA_R_BAD_Q_VALUE);
+        *ret = FFC_CHECK_INVALID_PQ;
+        return 0;
+    }
+
+    return 1;
+}
+
 int ossl_dsa_check_params(const DSA *dsa, int checktype, int *ret)
 {
+    if (!dsa_precheck_params(dsa, ret))
+        return 0;
+
     if (checktype == OSSL_KEYMGMT_VALIDATE_QUICK_CHECK)
         return ossl_ffc_params_simple_validate(dsa->libctx, &dsa->params,
                                                FFC_PARAM_TYPE_DSA, ret);
@@ -39,6 +65,9 @@ int ossl_dsa_check_params(const DSA *dsa, int checktype, int *ret)
  */
 int ossl_dsa_check_pub_key(const DSA *dsa, const BIGNUM *pub_key, int *ret)
 {
+    if (!dsa_precheck_params(dsa, ret))
+        return 0;
+
     return ossl_ffc_validate_public_key(&dsa->params, pub_key, ret)
            && *ret == 0;
 }
@@ -50,6 +79,9 @@ int ossl_dsa_check_pub_key(const DSA *dsa, const BIGNUM *pub_key, int *ret)
  */
 int ossl_dsa_check_pub_key_partial(const DSA *dsa, const BIGNUM *pub_key, int *ret)
 {
+    if (!dsa_precheck_params(dsa, ret))
+        return 0;
+
     return ossl_ffc_validate_public_key_partial(&dsa->params, pub_key, ret)
            && *ret == 0;
 }
@@ -58,8 +90,10 @@ int ossl_dsa_check_priv_key(const DSA *dsa, const BIGNUM *priv_key, int *ret)
 {
     *ret = 0;
 
-    return (dsa->params.q != NULL
-            && ossl_ffc_validate_private_key(dsa->params.q, priv_key, ret));
+    if (!dsa_precheck_params(dsa, ret))
+        return 0;
+
+    return ossl_ffc_validate_private_key(dsa->params.q, priv_key, ret);
 }
 
 /*
@@ -72,8 +106,10 @@ int ossl_dsa_check_pairwise(const DSA *dsa)
     BN_CTX *ctx = NULL;
     BIGNUM *pub_key = NULL;
 
-    if (dsa->params.p == NULL
-        || dsa->params.g == NULL
+    if (!dsa_precheck_params(dsa, &ret))
+        return 0;
+
+    if (dsa->params.g == NULL
         || dsa->priv_key == NULL
         || dsa->pub_key == NULL)
         return 0;