Add a test for custom EVP_PKEY_METHODs

Adds a test for using custom EVP_PKEY_METHODs without an ENGINE. As part
of this we also test having a custom EVP_PKEY_METHOD that wraps a built-in
EVP_PKEY_METHOD. We do this for both legacy and provided keys.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16118)

1 files changed, 328 insertions, 0 deletions

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index f10e0077ec..5358a54a6d 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -400,6 +400,47 @@ static const unsigned char kExampleED25519PubKeyDER[] = {
     0x97, 0xa3, 0x08, 0xdc, 0x65, 0x80, 0x39, 0x29
 };
 
+# ifndef OPENSSL_NO_DEPRECATED_3_0
+static const unsigned char kExampleX25519KeyDER[] = {
+    0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6e,
+    0x04, 0x22, 0x04, 0x20, 0xa0, 0x24, 0x3a, 0x31, 0x24, 0xc3, 0x3f, 0xf6,
+    0x7b, 0x96, 0x0b, 0xd4, 0x8f, 0xd1, 0xee, 0x67, 0xf2, 0x9b, 0x88, 0xac,
+    0x50, 0xce, 0x97, 0x36, 0xdd, 0xaf, 0x25, 0xf6, 0x10, 0x34, 0x96, 0x6e
+};
+# endif
+#endif
+
+/* kExampleDHKeyDER is a DH private key in ASN.1, DER format. */
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+# ifndef OPENSSL_NO_DH
+static const unsigned char kExampleDHKeyDER[] = {
+    0x30, 0x82, 0x01, 0x21, 0x02, 0x01, 0x00, 0x30, 0x81, 0x95, 0x06, 0x09,
+    0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x03, 0x01, 0x30, 0x81, 0x87,
+    0x02, 0x81, 0x81, 0x00, 0xf7, 0x52, 0xc2, 0x68, 0xcc, 0x66, 0xc4, 0x8d,
+    0x03, 0x3f, 0xfa, 0x9c, 0x52, 0xd0, 0xd8, 0x33, 0xf2, 0xe1, 0xc9, 0x9e,
+    0xb7, 0xe7, 0x6e, 0x90, 0x97, 0xeb, 0x92, 0x91, 0x6a, 0x9a, 0x85, 0x63,
+    0x92, 0x79, 0xab, 0xb6, 0x3d, 0x23, 0x58, 0x5a, 0xe8, 0x45, 0x06, 0x81,
+    0x97, 0x77, 0xe1, 0xcc, 0x34, 0x4e, 0xae, 0x36, 0x80, 0xf2, 0xc4, 0x7f,
+    0x8a, 0x52, 0xb8, 0xdb, 0x58, 0xc8, 0x4b, 0x12, 0x4c, 0xf1, 0x4c, 0x53,
+    0xc1, 0x89, 0x39, 0x8d, 0xb6, 0x06, 0xd8, 0xea, 0x7f, 0x2d, 0x36, 0x53,
+    0x96, 0x29, 0xbe, 0xb6, 0x75, 0xfc, 0xe7, 0xf3, 0x36, 0xd6, 0xf4, 0x8f,
+    0x16, 0xa6, 0xc7, 0xec, 0x7b, 0xce, 0x42, 0x8d, 0x48, 0x2e, 0xb7, 0x74,
+    0x00, 0x11, 0x52, 0x61, 0xb4, 0x19, 0x35, 0xec, 0x5c, 0xe4, 0xbe, 0x34,
+    0xc6, 0x59, 0x64, 0x5e, 0x42, 0x61, 0x70, 0x54, 0xf4, 0xe9, 0x6b, 0x53,
+    0x02, 0x01, 0x02, 0x04, 0x81, 0x83, 0x02, 0x81, 0x80, 0x64, 0xc2, 0xe3,
+    0x09, 0x69, 0x37, 0x3c, 0xd2, 0x4a, 0xba, 0xc3, 0x78, 0x6a, 0x9b, 0x8a,
+    0x2a, 0xdb, 0xe7, 0xe6, 0xc0, 0xfa, 0x3a, 0xbe, 0x39, 0x67, 0xc0, 0xa9,
+    0x2a, 0xf0, 0x0a, 0xc1, 0x53, 0x1c, 0xdb, 0xfa, 0x1a, 0x26, 0x98, 0xb0,
+    0x8c, 0xc6, 0x06, 0x4a, 0xa2, 0x48, 0xd3, 0xa4, 0x3b, 0xbd, 0x05, 0x48,
+    0xea, 0x59, 0xdb, 0x18, 0xa4, 0xca, 0x66, 0xd9, 0x5d, 0xb8, 0x95, 0xd1,
+    0xeb, 0x97, 0x3d, 0x66, 0x97, 0x5c, 0x86, 0x8f, 0x7e, 0x90, 0xd3, 0x43,
+    0xd1, 0xa2, 0x0d, 0xcb, 0xe7, 0xeb, 0x90, 0xea, 0x09, 0x40, 0xb1, 0x6f,
+    0xf7, 0x4c, 0xf2, 0x41, 0x83, 0x1d, 0xd0, 0x76, 0xef, 0xaf, 0x55, 0x6f,
+    0x5d, 0xa9, 0xa3, 0x55, 0x81, 0x2a, 0xd1, 0x5d, 0x9d, 0x22, 0x77, 0x97,
+    0x83, 0xde, 0xad, 0xb6, 0x5d, 0x19, 0xc1, 0x53, 0xec, 0xfb, 0xaf, 0x06,
+    0x2e, 0x87, 0x2a, 0x0b, 0x7a
+};
+# endif
 #endif
 
 static const unsigned char kCFBDefaultKey[] = {
@@ -558,6 +599,36 @@ static EVP_PKEY *load_example_dsa_key(void)
 }
 #endif
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+# ifndef OPENSSL_NO_DH
+static EVP_PKEY *load_example_dh_key(void)
+{
+    return load_example_key("DH", kExampleDHKeyDER,
+                            sizeof(kExampleDHKeyDER));
+}
+# endif
+
+# ifndef OPENSSL_NO_EC
+static EVP_PKEY *load_example_ec_key(void)
+{
+    return load_example_key("EC", kExampleECKeyDER,
+                            sizeof(kExampleECKeyDER));
+}
+
+static EVP_PKEY *load_example_ed25519_key(void)
+{
+    return load_example_key("ED25519", kExampleED25519KeyDER,
+                            sizeof(kExampleED25519KeyDER));
+}
+
+static EVP_PKEY *load_example_x25519_key(void)
+{
+    return load_example_key("X25519", kExampleX25519KeyDER,
+                            sizeof(kExampleX25519KeyDER));
+}
+# endif
+#endif /* OPENSSL_NO_DEPRECATED_3_0 */
+
 static EVP_PKEY *load_example_hmac_key(void)
 {
     EVP_PKEY *pkey = NULL;
@@ -3367,6 +3438,259 @@ static int test_gcm_reinit(int idx)
     return testresult;
 }
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+static EVP_PKEY_METHOD *custom_pmeth =  NULL;
+static const EVP_PKEY_METHOD *orig_pmeth = NULL;
+
+# define EVP_PKEY_CTRL_MY_COMMAND 9999
+
+static int custom_pmeth_init(EVP_PKEY_CTX *ctx)
+{
+    int (*pinit)(EVP_PKEY_CTX *ctx);
+
+    EVP_PKEY_meth_get_init(orig_pmeth, &pinit);
+    return pinit(ctx);
+}
+
+static void custom_pmeth_cleanup(EVP_PKEY_CTX *ctx)
+{
+    void (*pcleanup)(EVP_PKEY_CTX *ctx);
+
+    EVP_PKEY_meth_get_cleanup(orig_pmeth, &pcleanup);
+    pcleanup(ctx);
+}
+
+static int custom_pmeth_sign(EVP_PKEY_CTX *ctx, unsigned char *out,
+                             size_t *outlen, const unsigned char *in,
+                             size_t inlen)
+{
+    int (*psign)(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
+                 const unsigned char *tbs, size_t tbslen);
+
+    EVP_PKEY_meth_get_sign(orig_pmeth, NULL, &psign);
+    return psign(ctx, out, outlen, in, inlen);
+}
+
+static int custom_pmeth_digestsign(EVP_MD_CTX *ctx, unsigned char *sig,
+                                   size_t *siglen, const unsigned char *tbs,
+                                   size_t tbslen)
+{
+    int (*pdigestsign)(EVP_MD_CTX *ctx, unsigned char *sig, size_t *siglen,
+                       const unsigned char *tbs, size_t tbslen);
+
+    EVP_PKEY_meth_get_digestsign((EVP_PKEY_METHOD *)orig_pmeth, &pdigestsign);
+    return pdigestsign(ctx, sig, siglen, tbs, tbslen);
+}
+
+static int custom_pmeth_derive(EVP_PKEY_CTX *ctx, unsigned char *key,
+                               size_t *keylen)
+{
+    int (*pderive)(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen);
+
+    EVP_PKEY_meth_get_derive(orig_pmeth, NULL, &pderive);
+    return pderive(ctx, key, keylen);
+}
+
+static int custom_pmeth_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src)
+{
+    int (*pcopy)(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src);
+
+    EVP_PKEY_meth_get_copy(orig_pmeth, &pcopy);
+    return pcopy(dst, src);
+}
+
+static int ctrl_called;
+
+static int custom_pmeth_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
+{
+    int (*pctrl)(EVP_PKEY_CTX *ctx, int type, int p1, void *p2);
+
+    EVP_PKEY_meth_get_ctrl(orig_pmeth, &pctrl, NULL);
+
+    if (type == EVP_PKEY_CTRL_MY_COMMAND) {
+        ctrl_called = 1;
+        return 1;
+    }
+
+    return pctrl(ctx, type, p1, p2);
+}
+
+static int test_custom_pmeth(int idx)
+{
+    EVP_PKEY_CTX *pctx = NULL;
+    EVP_MD_CTX *ctx = NULL;
+    EVP_PKEY *pkey = NULL;
+    int id, orig_id, orig_flags;
+    int testresult = 0;
+    size_t reslen;
+    unsigned char *res = NULL;
+    unsigned char msg[] = { 'H', 'e', 'l', 'l', 'o' };
+    const EVP_MD *md = EVP_sha256();
+    int doderive = 0;
+
+    ctrl_called = 0;
+
+    /* We call deprecated APIs so this test doesn't support a custom libctx */
+    if (testctx != NULL)
+        return 1;
+
+    switch(idx) {
+    case 0:
+    case 6:
+        id = EVP_PKEY_RSA;
+        pkey = load_example_rsa_key();
+        break;
+    case 1:
+    case 7:
+# ifndef OPENSSL_NO_DSA
+        id = EVP_PKEY_DSA;
+        pkey = load_example_dsa_key();
+        break;
+# else
+        return 1;
+# endif
+    case 2:
+    case 8:
+# ifndef OPENSSL_NO_EC
+        id = EVP_PKEY_EC;
+        pkey = load_example_ec_key();
+        break;
+# else
+        return 1;
+# endif
+    case 3:
+    case 9:
+# ifndef OPENSSL_NO_EC
+        id = EVP_PKEY_ED25519;
+        md = NULL;
+        pkey = load_example_ed25519_key();
+        break;
+# else
+        return 1;
+# endif
+    case 4:
+    case 10:
+# ifndef OPENSSL_NO_DH
+        id = EVP_PKEY_DH;
+        doderive = 1;
+        pkey = load_example_dh_key();
+        break;
+# else
+        return 1;
+# endif
+    case 5:
+    case 11:
+# ifndef OPENSSL_NO_EC
+        id = EVP_PKEY_X25519;
+        doderive = 1;
+        pkey = load_example_x25519_key();
+        break;
+# else
+        return 1;
+# endif
+    default:
+        TEST_error("Should not happen");
+        goto err;
+    }
+
+    if (!TEST_ptr(pkey))
+        goto err;
+
+    if (idx < 6) {
+        if (!TEST_true(evp_pkey_is_provided(pkey)))
+            goto err;
+    } else {
+        EVP_PKEY *tmp = pkey;
+
+        /* Convert to a legacy key */
+        pkey = EVP_PKEY_new();
+        if (!TEST_ptr(pkey)) {
+            pkey = tmp;
+            goto err;
+        }
+        if (!TEST_true(evp_pkey_copy_downgraded(&pkey, tmp))) {
+            EVP_PKEY_free(tmp);
+            goto err;
+        }
+        EVP_PKEY_free(tmp);
+        if (!TEST_true(evp_pkey_is_legacy(pkey)))
+            goto err;
+    }
+
+    if (!TEST_ptr(orig_pmeth = EVP_PKEY_meth_find(id))
+            || !TEST_ptr(pkey))
+        goto err;
+
+    EVP_PKEY_meth_get0_info(&orig_id, &orig_flags, orig_pmeth);
+    if (!TEST_int_eq(orig_id, id)
+            || !TEST_ptr(custom_pmeth = EVP_PKEY_meth_new(id, orig_flags)))
+        goto err;
+
+    if (id == EVP_PKEY_ED25519) {
+        EVP_PKEY_meth_set_digestsign(custom_pmeth, custom_pmeth_digestsign);
+    } if (id == EVP_PKEY_DH || id == EVP_PKEY_X25519) {
+        EVP_PKEY_meth_set_derive(custom_pmeth, NULL, custom_pmeth_derive);
+    } else {
+        EVP_PKEY_meth_set_sign(custom_pmeth, NULL, custom_pmeth_sign);
+    }
+    if (id != EVP_PKEY_ED25519 && id != EVP_PKEY_X25519) {
+        EVP_PKEY_meth_set_init(custom_pmeth, custom_pmeth_init);
+        EVP_PKEY_meth_set_cleanup(custom_pmeth, custom_pmeth_cleanup);
+        EVP_PKEY_meth_set_copy(custom_pmeth, custom_pmeth_copy);
+    }
+    EVP_PKEY_meth_set_ctrl(custom_pmeth, custom_pmeth_ctrl, NULL);
+    if (!TEST_true(EVP_PKEY_meth_add0(custom_pmeth)))
+        goto err;
+
+    if (doderive) {
+        pctx = EVP_PKEY_CTX_new(pkey, NULL);
+        if (!TEST_ptr(pctx)
+                || !TEST_int_eq(EVP_PKEY_derive_init(pctx), 1)
+                || !TEST_int_ge(EVP_PKEY_CTX_ctrl(pctx, -1, -1,
+                                                EVP_PKEY_CTRL_MY_COMMAND, 0, NULL),
+                                1)
+                || !TEST_int_eq(ctrl_called, 1)
+                || !TEST_int_ge(EVP_PKEY_derive_set_peer(pctx, pkey), 1)
+                || !TEST_int_ge(EVP_PKEY_derive(pctx, NULL, &reslen), 1)
+                || !TEST_ptr(res = OPENSSL_malloc(reslen))
+                || !TEST_int_ge(EVP_PKEY_derive(pctx, res, &reslen), 1))
+            goto err;
+    } else {
+        ctx = EVP_MD_CTX_new();
+        reslen = EVP_PKEY_size(pkey);
+        res = OPENSSL_malloc(reslen);
+        if (!TEST_ptr(ctx)
+                || !TEST_ptr(res)
+                || !TEST_true(EVP_DigestSignInit(ctx, &pctx, md, NULL, pkey))
+                || !TEST_int_ge(EVP_PKEY_CTX_ctrl(pctx, -1, -1,
+                                                EVP_PKEY_CTRL_MY_COMMAND, 0, NULL),
+                                1)
+                || !TEST_int_eq(ctrl_called, 1))
+            goto err;
+
+        if (id == EVP_PKEY_ED25519) {
+            if (!TEST_true(EVP_DigestSign(ctx, res, &reslen, msg, sizeof(msg))))
+                goto err;
+        } else {
+            if (!TEST_true(EVP_DigestUpdate(ctx, msg, sizeof(msg)))
+                    || !TEST_true(EVP_DigestSignFinal(ctx, res, &reslen)))
+                goto err;
+        }
+    }
+
+    testresult = 1;
+ err:
+    OPENSSL_free(res);
+    EVP_MD_CTX_free(ctx);
+    if (doderive)
+        EVP_PKEY_CTX_free(pctx);
+    EVP_PKEY_free(pkey);
+    EVP_PKEY_meth_remove(custom_pmeth);
+    EVP_PKEY_meth_free(custom_pmeth);
+    custom_pmeth = NULL;
+    return testresult;
+}
+#endif
 
 typedef enum OPTION_choice {
     OPT_ERR = -1,
@@ -3488,6 +3812,10 @@ int setup_tests(void)
     ADD_ALL_TESTS(test_evp_reset, OSSL_NELEM(evp_reset_tests));
     ADD_ALL_TESTS(test_gcm_reinit, OSSL_NELEM(gcm_reinit_tests));
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    ADD_ALL_TESTS(test_custom_pmeth, 12);
+#endif
+
     return 1;
 }