DH_check_pub_key() should not fail when setting result code

The semantics of ossl_ffc_validate_public_key() and
ossl_ffc_validate_public_key_partial() needs to be changed
to not return error on non-fatal problems.

Fixes #22287

Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22291)

1 files changed, 18 insertions, 20 deletions

diff --git a/test/ffc_internal_test.c b/test/ffc_internal_test.c
index 0332e777c0..c56d1d0e99 100644
--- a/test/ffc_internal_test.c
+++ b/test/ffc_internal_test.c
@@ -455,22 +455,20 @@ static int ffc_public_validate_test(void)
     if (!TEST_true(BN_set_word(pub, 1)))
         goto err;
     BN_set_negative(pub, 1);
-    /* Fail if public key is negative */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if public key is negative */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_SMALL, res))
         goto err;
     if (!TEST_true(BN_set_word(pub, 0)))
         goto err;
-    if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_SMALL, res))
-        goto err;
-    /* Fail if public key is zero */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if public key is zero */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_SMALL, res))
         goto err;
-    /* Fail if public key is 1 */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, BN_value_one(), &res)))
+    /* Check must succeed but set res if public key is 1 */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, BN_value_one(), &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_SMALL, res))
         goto err;
@@ -482,24 +480,24 @@ static int ffc_public_validate_test(void)
 
     if (!TEST_ptr(BN_copy(pub, params->p)))
         goto err;
-    /* Fail if public key = p */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if public key = p */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_LARGE, res))
         goto err;
 
     if (!TEST_true(BN_sub_word(pub, 1)))
         goto err;
-    /* Fail if public key = p - 1 */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if public key = p - 1 */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_TOO_LARGE, res))
         goto err;
 
     if (!TEST_true(BN_sub_word(pub, 1)))
         goto err;
-    /* Fail if public key is not related to p & q */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if public key is not related to p & q */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PUBKEY_INVALID, res))
         goto err;
@@ -510,14 +508,14 @@ static int ffc_public_validate_test(void)
     if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
 
-    /* Fail if params is NULL */
-    if (!TEST_false(ossl_ffc_validate_public_key(NULL, pub, &res)))
+    /* Check must succeed but set res if params is NULL */
+    if (!TEST_true(ossl_ffc_validate_public_key(NULL, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PASSED_NULL_PARAM, res))
         goto err;
     res = -1;
-    /* Fail if pubkey is NULL */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, NULL, &res)))
+    /* Check must succeed but set res if pubkey is NULL */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, NULL, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PASSED_NULL_PARAM, res))
         goto err;
@@ -525,8 +523,8 @@ static int ffc_public_validate_test(void)
 
     BN_free(params->p);
     params->p = NULL;
-    /* Fail if params->p is NULL */
-    if (!TEST_false(ossl_ffc_validate_public_key(params, pub, &res)))
+    /* Check must succeed but set res if params->p is NULL */
+    if (!TEST_true(ossl_ffc_validate_public_key(params, pub, &res)))
         goto err;
     if (!TEST_int_eq(FFC_ERROR_PASSED_NULL_PARAM, res))
         goto err;