Add coverage test for ossl_rsa_sp800_56b_derive_params_from_pq

This test runs the error path for the above function.

Reviewed-by: Hugo Landau <hlandau@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/20200)

1 files changed, 40 insertions, 1 deletions

diff --git a/test/rsa_sp800_56b_test.c b/test/rsa_sp800_56b_test.c
index b17dafe399..10443683b9 100644
--- a/test/rsa_sp800_56b_test.c
+++ b/test/rsa_sp800_56b_test.c
@@ -300,7 +300,8 @@ static int test_check_crt_components(void)
         BN_free(q);
         goto end;
     }
-    ret = TEST_true(ossl_rsa_sp800_56b_derive_params_from_pq(key, 8, e, ctx))
+
+    ret = TEST_int_eq(ossl_rsa_sp800_56b_derive_params_from_pq(key, 8, e, ctx), 1)
           && TEST_BN_eq_word(key->n, N)
           && TEST_BN_eq_word(key->dmp1, DP)
           && TEST_BN_eq_word(key->dmq1, DQ)
@@ -345,6 +346,43 @@ end:
     return ret;
 }
 
+static const struct derive_from_pq_test {
+    int p, q, e;
+} derive_from_pq_tests[] = {
+    { 15, 17, 6 }, /* Mod_inverse failure */
+    { 0, 17, 5 }, /* d is too small */
+};
+
+static int test_derive_params_from_pq_fail(int tst)
+{
+    int ret = 0;
+    RSA *key = NULL;
+    BN_CTX *ctx = NULL;
+    BIGNUM *p = NULL, *q = NULL, *e = NULL;
+
+    ret = TEST_ptr(key = RSA_new())
+          && TEST_ptr(ctx = BN_CTX_new())
+          && TEST_ptr(p = BN_new())
+          && TEST_ptr(q = BN_new())
+          && TEST_ptr(e = BN_new())
+          && TEST_true(BN_set_word(p, derive_from_pq_tests[tst].p))
+          && TEST_true(BN_set_word(q, derive_from_pq_tests[tst].q))
+          && TEST_true(BN_set_word(e, derive_from_pq_tests[tst].e))
+          && TEST_true(RSA_set0_factors(key, p, q));
+    if (!ret) {
+        BN_free(p);
+        BN_free(q);
+        goto end;
+    }
+
+    ret = TEST_int_le(ossl_rsa_sp800_56b_derive_params_from_pq(key, 8, e, ctx), 0);
+end:
+    BN_free(e);
+    RSA_free(key);
+    BN_CTX_free(ctx);
+    return ret;
+}
+
 static int test_pq_diff(void)
 {
     int ret = 0;
@@ -539,6 +577,7 @@ int setup_tests(void)
     ADD_TEST(test_check_prime_factor);
     ADD_TEST(test_check_private_exponent);
     ADD_TEST(test_check_crt_components);
+    ADD_ALL_TESTS(test_derive_params_from_pq_fail, (int)OSSL_NELEM(derive_from_pq_tests));
     ADD_TEST(test_check_private_key);
     ADD_TEST(test_check_public_key);
     ADD_TEST(test_invalid_keypair);