test/endecode_test.c: Update to specify output structures

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13167)

1 files changed, 65 insertions, 82 deletions

diff --git a/test/endecode_test.c b/test/endecode_test.c
index 57ba6774e4..4a86e71b76 100644
--- a/test/endecode_test.c
+++ b/test/endecode_test.c
@@ -89,8 +89,9 @@ static EVP_PKEY *make_key(const char *type, EVP_PKEY *template,
  * (TEST_strn2_eq(), for example).
  */
 
-typedef int (encoder)(void **encoded, long *encoded_len, void *object,
-                      const char *output_type, int selection,
+typedef int (encoder)(void **encoded, long *encoded_len,
+                      void *object, int selection,
+                      const char *output_type, const char *output_structure,
                       const char *pass, const char *pcipher);
 typedef int (decoder)(void **object, void *encoded, long encoded_len,
                       const char *keytype, const char *input_type,
@@ -103,7 +104,8 @@ typedef void (dumper)(const char *label, const void *data, size_t data_len);
 #define FLAG_DECODE_WITH_TYPE   0x0001
 
 static int test_encode_decode(const char *type, EVP_PKEY *pkey,
-                              const char *output_type, int selection,
+                              int selection, const char *output_type,
+                              const char *output_structure,
                               const char *pass, const char *pcipher,
                               encoder *encode_cb, decoder *decode_cb,
                               tester *test_cb, checker *check_cb,
@@ -121,14 +123,14 @@ static int test_encode_decode(const char *type, EVP_PKEY *pkey,
      * encoding |pkey2| as well.  That last encoding is for checking and
      * dumping purposes.
      */
-    if (!TEST_true(encode_cb(&encoded, &encoded_len, pkey, output_type,
-                             selection, pass, pcipher))
+    if (!TEST_true(encode_cb(&encoded, &encoded_len, pkey, selection,
+                             output_type, output_structure, pass, pcipher))
         || !TEST_true(check_cb(type, encoded, encoded_len))
         || !TEST_true(decode_cb((void **)&pkey2, encoded, encoded_len,
                                 (flags & FLAG_DECODE_WITH_TYPE ? type : NULL),
                                 output_type, selection, pass))
-        || !TEST_true(encode_cb(&encoded2, &encoded2_len, pkey2, output_type,
-                                selection, pass, pcipher)))
+        || !TEST_true(encode_cb(&encoded2, &encoded2_len, pkey2, selection,
+                                output_type, output_structure, pass, pcipher)))
         goto end;
 
     if (selection == OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS) {
@@ -163,11 +165,12 @@ static int test_encode_decode(const char *type, EVP_PKEY *pkey,
     return ok;
 }
 
-/* Encoding and desencoding methods */
+/* Encoding and decoding methods */
 
 static int encode_EVP_PKEY_prov(void **encoded, long *encoded_len,
-                                void *object,
-                                const char *output_type, int selection,
+                                void *object, int selection,
+                                const char *output_type,
+                                const char *output_structure,
                                 const char *pass, const char *pcipher)
 {
     EVP_PKEY *pkey = object;
@@ -177,9 +180,11 @@ static int encode_EVP_PKEY_prov(void **encoded, long *encoded_len,
     const unsigned char *upass = (const unsigned char *)pass;
     int ok = 0;
 
-    if (!TEST_ptr(ectx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, output_type,
-                                                          selection,
+    if (!TEST_ptr(ectx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection,
+                                                          output_type,
+                                                          output_structure,
                                                           NULL, NULL))
+        || !TEST_int_gt(OSSL_ENCODER_CTX_get_num_encoders(ectx), 0)
         || (pass != NULL
             && !TEST_true(OSSL_ENCODER_CTX_set_passphrase(ectx, upass,
                                                           strlen(pass))))
@@ -275,8 +280,9 @@ static int decode_EVP_PKEY_prov(void **object, void *encoded, long encoded_len,
 }
 
 static int encode_EVP_PKEY_legacy_PEM(void **encoded, long *encoded_len,
-                                      void *object,
-                                      const char *output_type, int selection,
+                                      void *object, ossl_unused int selection,
+                                      ossl_unused const char *output_type,
+                                      ossl_unused const char *output_structure,
                                       const char *pass, const char *pcipher)
 {
     EVP_PKEY *pkey = object;
@@ -314,9 +320,9 @@ static int encode_EVP_PKEY_legacy_PEM(void **encoded, long *encoded_len,
 
 #ifndef OPENSSL_NO_DSA
 static int encode_EVP_PKEY_MSBLOB(void **encoded, long *encoded_len,
-                                  void *object,
+                                  void *object, int selection,
                                   ossl_unused const char *output_type,
-                                  ossl_unused int selection,
+                                  ossl_unused const char *output_structure,
                                   ossl_unused const char *pass,
                                   ossl_unused const char *pcipher)
 {
@@ -325,38 +331,18 @@ static int encode_EVP_PKEY_MSBLOB(void **encoded, long *encoded_len,
     BUF_MEM *mem_buf = NULL;
     int ok = 0;
 
-    if (!TEST_ptr(mem_ser = BIO_new(BIO_s_mem()))
-        || !TEST_int_ge(i2b_PrivateKey_bio(mem_ser, pkey), 0)
-        || !TEST_true(BIO_get_mem_ptr(mem_ser, &mem_buf) > 0)
-        || !TEST_ptr(*encoded = mem_buf->data)
-        || !TEST_long_gt(*encoded_len = mem_buf->length, 0))
+    if (!TEST_ptr(mem_ser = BIO_new(BIO_s_mem())))
         goto end;
 
-    /* Detach the encoded output */
-    mem_buf->data = NULL;
-    mem_buf->length = 0;
-    ok = 1;
- end:
-    BIO_free(mem_ser);
-    return ok;
-}
-
-static int encode_public_EVP_PKEY_MSBLOB(void **encoded,
-                                         long *encoded_len,
-                                         void *object,
-                                         ossl_unused const char *output_type,
-                                         ossl_unused int selection,
-                                         ossl_unused const char *pass,
-                                         ossl_unused const char *pcipher)
-{
-    EVP_PKEY *pkey = object;
-    BIO *mem_ser = NULL;
-    BUF_MEM *mem_buf = NULL;
-    int ok = 0;
+    if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) {
+        if (!TEST_int_ge(i2b_PrivateKey_bio(mem_ser, pkey), 0))
+            goto end;
+    } else {
+        if (!TEST_int_ge(i2b_PublicKey_bio(mem_ser, pkey), 0))
+            goto end;
+    }
 
-    if (!TEST_ptr(mem_ser = BIO_new(BIO_s_mem()))
-        || !TEST_int_ge(i2b_PublicKey_bio(mem_ser, pkey), 0)
-        || !TEST_true(BIO_get_mem_ptr(mem_ser, &mem_buf) > 0)
+    if (!TEST_true(BIO_get_mem_ptr(mem_ser, &mem_buf) > 0)
         || !TEST_ptr(*encoded = mem_buf->data)
         || !TEST_long_gt(*encoded_len = mem_buf->length, 0))
         goto end;
@@ -379,9 +365,9 @@ static int pass_pw(char *buf, int size, int rwflag, void *userdata)
 }
 
 static int encode_EVP_PKEY_PVK(void **encoded, long *encoded_len,
-                               void *object,
+                               void *object, int selection,
                                ossl_unused const char *output_type,
-                               ossl_unused int selection,
+                               ossl_unused const char *output_structure,
                                const char *pass,
                                ossl_unused const char *pcipher)
 {
@@ -391,7 +377,9 @@ static int encode_EVP_PKEY_PVK(void **encoded, long *encoded_len,
     int enc = (pass != NULL);
     int ok = 0;
 
-    if (!TEST_ptr(mem_ser = BIO_new(BIO_s_mem()))
+    if (!TEST_true(ossl_assert((selection
+                                & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0))
+        || !TEST_ptr(mem_ser = BIO_new(BIO_s_mem()))
         || !TEST_int_ge(i2b_PVK_bio(mem_ser, pkey, enc,
                                     pass_pw, (void *)pass), 0)
         || !TEST_true(BIO_get_mem_ptr(mem_ser, &mem_buf) > 0)
@@ -486,9 +474,9 @@ static int check_unprotected_PKCS8_DER(const char *type,
 static int test_unprotected_via_DER(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "DER", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "DER", "pkcs8", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_mem, check_unprotected_PKCS8_DER,
                               dump_der, 0);
@@ -506,10 +494,9 @@ static int check_unprotected_PKCS8_PEM(const char *type,
 
 static int test_unprotected_via_PEM(const char *type, EVP_PKEY *key)
 {
-    return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_KEYPAIR
+    return test_encode_decode(type, key, OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "PEM", "pkcs8", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_text, check_unprotected_PKCS8_PEM,
                               dump_pem, 0);
@@ -554,9 +541,8 @@ static int check_params_PEM(const char *type,
 
 static int test_params_via_DER(const char *type, EVP_PKEY *key)
 {
-    return test_encode_decode(type, key,
-                              "DER", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+    return test_encode_decode(type, key, OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
+                              "DER", "type-specific", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_mem, check_params_DER,
                               dump_der, FLAG_DECODE_WITH_TYPE);
@@ -564,9 +550,8 @@ static int test_params_via_DER(const char *type, EVP_PKEY *key)
 
 static int test_params_via_PEM(const char *type, EVP_PKEY *key)
 {
-    return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+    return test_encode_decode(type, key, OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
+                              "PEM", "type-specific", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_text, check_params_PEM,
                               dump_pem, 0);
@@ -587,9 +572,9 @@ static int check_unprotected_legacy_PEM(const char *type,
 static int test_unprotected_via_legacy_PEM(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "PEM", "type-specific", NULL, NULL,
                               encode_EVP_PKEY_legacy_PEM, decode_EVP_PKEY_prov,
                               test_text, check_unprotected_legacy_PEM,
                               dump_pem, 0);
@@ -609,9 +594,9 @@ static int check_MSBLOB(const char *type, const void *data, size_t data_len)
 static int test_unprotected_via_MSBLOB(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "MSBLOB", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "MSBLOB", NULL, NULL, NULL,
                               encode_EVP_PKEY_MSBLOB, decode_EVP_PKEY_prov,
                               test_mem, check_MSBLOB,
                               dump_der, 0);
@@ -630,9 +615,9 @@ static int check_PVK(const char *type, const void *data, size_t data_len)
 static int test_unprotected_via_PVK(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PVK", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "PVK", NULL, NULL, NULL,
                               encode_EVP_PKEY_PVK, decode_EVP_PKEY_prov,
                               test_mem, check_PVK,
                               dump_der, 0);
@@ -657,9 +642,9 @@ static int check_protected_PKCS8_DER(const char *type,
 static int test_protected_via_DER(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "DER", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              pass, pass_cipher,
+                              "DER", "pkcs8", pass, pass_cipher,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_mem, check_protected_PKCS8_DER,
                               dump_der, 0);
@@ -678,9 +663,9 @@ static int check_protected_PKCS8_PEM(const char *type,
 static int test_protected_via_PEM(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              pass, pass_cipher,
+                              "PEM", "pkcs8", pass, pass_cipher,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_text, check_protected_PKCS8_PEM,
                               dump_pem, 0);
@@ -702,9 +687,9 @@ static int check_protected_legacy_PEM(const char *type,
 static int test_protected_via_legacy_PEM(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              pass, pass_cipher,
+                              "PEM", "type-specific", pass, pass_cipher,
                               encode_EVP_PKEY_legacy_PEM, decode_EVP_PKEY_prov,
                               test_text, check_protected_legacy_PEM,
                               dump_pem, 0);
@@ -714,9 +699,9 @@ static int test_protected_via_legacy_PEM(const char *type, EVP_PKEY *key)
 static int test_protected_via_PVK(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PVK", OSSL_KEYMGMT_SELECT_KEYPAIR
+                              OSSL_KEYMGMT_SELECT_KEYPAIR
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              pass, NULL,
+                              "PVK", NULL, pass, NULL,
                               encode_EVP_PKEY_PVK, decode_EVP_PKEY_prov,
                               test_mem, check_PVK, dump_der, 0);
 }
@@ -735,9 +720,9 @@ static int check_public_DER(const char *type, const void *data, size_t data_len)
 static int test_public_via_DER(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "DER", OSSL_KEYMGMT_SELECT_PUBLIC_KEY
+                              OSSL_KEYMGMT_SELECT_PUBLIC_KEY
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "DER", "SubjectPublicKeyInfo", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_mem, check_public_DER, dump_der, 0);
 }
@@ -755,9 +740,9 @@ static int check_public_PEM(const char *type, const void *data, size_t data_len)
 static int test_public_via_PEM(const char *type, EVP_PKEY *key)
 {
     return test_encode_decode(type, key,
-                              "PEM", OSSL_KEYMGMT_SELECT_PUBLIC_KEY
+                              OSSL_KEYMGMT_SELECT_PUBLIC_KEY
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
+                              "PEM", "SubjectPublicKeyInfo", NULL, NULL,
                               encode_EVP_PKEY_prov, decode_EVP_PKEY_prov,
                               test_text, check_public_PEM, dump_pem, 0);
 }
@@ -776,12 +761,10 @@ static int check_public_MSBLOB(const char *type,
 
 static int test_public_via_MSBLOB(const char *type, EVP_PKEY *key)
 {
-    return test_encode_decode(type, key,
-                              "MSBLOB", OSSL_KEYMGMT_SELECT_PUBLIC_KEY
+    return test_encode_decode(type, key, OSSL_KEYMGMT_SELECT_PUBLIC_KEY
                               | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                              NULL, NULL,
-                              encode_public_EVP_PKEY_MSBLOB,
-                              decode_EVP_PKEY_prov,
+                              "MSBLOB", NULL, NULL, NULL,
+                              encode_EVP_PKEY_MSBLOB, decode_EVP_PKEY_prov,
                               test_mem, check_public_MSBLOB, dump_der, 0);
 }
 #endif