ENCODER: Add support for specifying the outermost output structure

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13167)

7 files changed, 135 insertions, 48 deletions

diff --git a/crypto/encode_decode/encoder_lib.c b/crypto/encode_decode/encoder_lib.c
index 593483313c..5e4bf69787 100644
--- a/crypto/encode_decode/encoder_lib.c
+++ b/crypto/encode_decode/encoder_lib.c
@@ -97,6 +97,22 @@ int OSSL_ENCODER_to_data(OSSL_ENCODER_CTX *ctx, unsigned char **pdata,
     return ret;
 }
 
+int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection)
+{
+    if (!ossl_assert(ctx != NULL)) {
+        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
+    if (!ossl_assert(selection != 0)) {
+        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_INVALID_ARGUMENT);
+        return 0;
+    }
+
+    ctx->selection = selection;
+    return 1;
+}
+
 int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
                                      const char *output_type)
 {
@@ -109,19 +125,15 @@ int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
     return 1;
 }
 
-int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection)
+int OSSL_ENCODER_CTX_set_output_structure(OSSL_ENCODER_CTX *ctx,
+                                          const char *output_structure)
 {
-    if (!ossl_assert(ctx != NULL)) {
+    if (!ossl_assert(ctx != NULL) || !ossl_assert(output_structure != NULL)) {
         ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
         return 0;
     }
 
-    if (!ossl_assert(selection != 0)) {
-        ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_INVALID_ARGUMENT);
-        return 0;
-    }
-
-    ctx->selection = selection;
+    ctx->output_structure = output_structure;
     return 1;
 }
 
@@ -129,7 +141,7 @@ static OSSL_ENCODER_INSTANCE *ossl_encoder_instance_new(OSSL_ENCODER *encoder,
                                                         void *encoderctx)
 {
     OSSL_ENCODER_INSTANCE *encoder_inst = NULL;
-    OSSL_PARAM params[3];
+    OSSL_PARAM params[4];
 
     if (!ossl_assert(encoder != NULL)) {
         ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER);
@@ -155,12 +167,16 @@ static OSSL_ENCODER_INSTANCE *ossl_encoder_instance_new(OSSL_ENCODER *encoder,
         OSSL_PARAM_construct_utf8_ptr(OSSL_ENCODER_PARAM_OUTPUT_TYPE,
                                       (char **)&encoder_inst->output_type, 0);
     params[1] =
+        OSSL_PARAM_construct_utf8_ptr(OSSL_ENCODER_PARAM_OUTPUT_STRUCTURE,
+                                      (char **)&encoder_inst->output_structure,
+                                      0);
+    params[2] =
         OSSL_PARAM_construct_utf8_ptr(OSSL_ENCODER_PARAM_INPUT_TYPE,
                                       (char **)&encoder_inst->input_type, 0);
-    params[2] = OSSL_PARAM_construct_end();
+    params[3] = OSSL_PARAM_construct_end();
 
     if (!encoder->get_params(params)
-        || !OSSL_PARAM_modified(&params[1]))
+        || !OSSL_PARAM_modified(&params[0]))
         goto err;
 
     if (!OSSL_ENCODER_up_ref(encoder)) {
@@ -312,6 +328,14 @@ OSSL_ENCODER_INSTANCE_get_output_type(OSSL_ENCODER_INSTANCE *encoder_inst)
     return encoder_inst->output_type;
 }
 
+const char *
+OSSL_ENCODER_INSTANCE_get_output_structure(OSSL_ENCODER_INSTANCE *encoder_inst)
+{
+    if (encoder_inst == NULL)
+        return NULL;
+    return encoder_inst->output_structure;
+}
+
 static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
 {
     size_t i, end;
@@ -319,6 +343,7 @@ static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
     size_t latest_output_length = 0;
     const char *latest_output_type = NULL;
     const char *last_input_type = NULL;
+    const char *last_output_structure = NULL;
     int ok = 0;
 
     end = OSSL_ENCODER_CTX_get_num_encoders(ctx);
@@ -331,10 +356,12 @@ static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
             OSSL_ENCODER_INSTANCE_get_input_type(encoder_inst);
         const char *current_output_type =
             OSSL_ENCODER_INSTANCE_get_output_type(encoder_inst);
+        const char *current_output_structure =
+            OSSL_ENCODER_INSTANCE_get_output_structure(encoder_inst);
         BIO *current_out;
         BIO *allocated_out = NULL;
         const void *current_data = NULL;
-        OSSL_PARAM abstract[3];
+        OSSL_PARAM abstract[10];
         OSSL_PARAM *abstract_p;
         const OSSL_PARAM *current_abstract = NULL;
 
@@ -374,6 +401,11 @@ static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
                 *abstract_p++ =
                     OSSL_PARAM_construct_utf8_string(OSSL_OBJECT_PARAM_DATA_TYPE,
                                                      (char *)last_input_type, 0);
+            if (last_output_structure != NULL)
+                *abstract_p++ =
+                    OSSL_PARAM_construct_utf8_string(OSSL_OBJECT_PARAM_DATA_STRUCTURE,
+                                                     (char *)last_output_structure,
+                                                     0);
             *abstract_p++ =
                 OSSL_PARAM_construct_octet_string(OSSL_OBJECT_PARAM_DATA,
                                                   latest_output,
@@ -398,6 +430,8 @@ static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
 
         if (current_input_type != NULL)
             last_input_type = current_input_type;
+        if (current_output_structure != NULL)
+            last_output_structure = current_output_structure;
 
         if (!ok)
             goto loop_end;
@@ -418,6 +452,8 @@ static int encoder_process(OSSL_ENCODER_CTX *ctx, BIO *out)
             BIO_free(allocated_out);
         }
 
+        latest_output_type = encoder_inst->output_type;
+
      loop_end:
         if (current_data != NULL)
             ctx->cleanup(ctx->construct_data);
diff --git a/crypto/encode_decode/encoder_local.h b/crypto/encode_decode/encoder_local.h
index 18cddf50fb..be4cba3783 100644
--- a/crypto/encode_decode/encoder_local.h
+++ b/crypto/encode_decode/encoder_local.h
@@ -52,10 +52,11 @@ struct ossl_decoder_st {
 };
 
 struct ossl_encoder_instance_st {
-    OSSL_ENCODER *encoder;       /* Never NULL */
-    void *encoderctx;            /* Never NULL */
-    const char *input_type;      /* May be NULL */
-    const char *output_type;     /* Never NULL */
+    OSSL_ENCODER *encoder;        /* Never NULL */
+    void *encoderctx;             /* Never NULL */
+    const char *input_type;       /* May be NULL */
+    const char *output_type;      /* Never NULL */
+    const char *output_structure; /* May be NULL */
 };
 
 DEFINE_STACK_OF(OSSL_ENCODER_INSTANCE)
@@ -64,17 +65,23 @@ void ossl_encoder_instance_free(OSSL_ENCODER_INSTANCE *encoder_inst);
 
 struct ossl_encoder_ctx_st {
     /*
-     * The desired output type.  The encoder implementation have a gettable
-     * "output-type" parameter that this will match against.
-     */
-    const char *output_type;
-    /*
      * Select what parts of an object will be encoded.  This selection is
      * bit encoded, and the bits correspond to selection bits available with
      * the provider side operation.  For example, when encoding an EVP_PKEY,
      * the OSSL_KEYMGMT_SELECT_ macros are used for this.
      */
     int selection;
+    /*
+     * The desired output type.  The encoder implementation must have a
+     * gettable "output-type" parameter that this will match against.
+     */
+    const char *output_type;
+    /*
+     * The desired output structure, if that's relevant for the type of
+     * object being encoded.  It may be used for selection of the starting
+     * encoder implementations in a chain.
+     */
+    const char *output_structure;
 
     /*
      * Decoders that are components of any current decoding path.
diff --git a/doc/man3/OSSL_ENCODER_CTX.pod b/doc/man3/OSSL_ENCODER_CTX.pod
index e8771fc087..d4ea01060f 100644
--- a/doc/man3/OSSL_ENCODER_CTX.pod
+++ b/doc/man3/OSSL_ENCODER_CTX.pod
@@ -7,8 +7,9 @@ OSSL_ENCODER_CTX_new,
 OSSL_ENCODER_settable_ctx_params,
 OSSL_ENCODER_CTX_set_params,
 OSSL_ENCODER_CTX_free,
-OSSL_ENCODER_CTX_set_output_type,
 OSSL_ENCODER_CTX_set_selection,
+OSSL_ENCODER_CTX_set_output_type,
+OSSL_ENCODER_CTX_set_output_structure,
 OSSL_ENCODER_CTX_add_encoder,
 OSSL_ENCODER_CTX_add_extra,
 OSSL_ENCODER_CTX_get_num_encoders,
@@ -17,6 +18,7 @@ OSSL_ENCODER_INSTANCE_get_encoder,
 OSSL_ENCODER_INSTANCE_get_encoder_ctx,
 OSSL_ENCODER_INSTANCE_get_input_type,
 OSSL_ENCODER_INSTANCE_get_output_type,
+OSSL_ENCODER_INSTANCE_get_output_structure,
 OSSL_ENCODER_CONSTRUCT,
 OSSL_ENCODER_CLEANUP,
 OSSL_ENCODER_CTX_set_construct,
@@ -36,9 +38,11 @@ OSSL_ENCODER_CTX_set_cleanup
                                  const OSSL_PARAM params[]);
  void OSSL_ENCODER_CTX_free(OSSL_ENCODER_CTX *ctx);
 
+ int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection);
  int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
                                       const char *output_type);
- int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection);
+ int OSSL_ENCODER_CTX_set_output_structure(OSSL_ENCODER_CTX *ctx,
+                                           const char *output_structure);
 
  int OSSL_ENCODER_CTX_add_encoder(OSSL_ENCODER_CTX *ctx, OSSL_ENCODER *encoder);
  int OSSL_ENCODER_CTX_add_extra(OSSL_ENCODER_CTX *ctx,
@@ -54,6 +58,8 @@ OSSL_ENCODER_CTX_set_cleanup
  OSSL_ENCODER_INSTANCE_get_input_type(OSSL_ENCODER_INSTANCE *encoder_inst);
  const char *
  OSSL_ENCODER_INSTANCE_get_output_type(OSSL_ENCODER_INSTANCE *encoder_inst);
+ const char *
+ OSSL_ENCODER_INSTANCE_get_output_structure(OSSL_ENCODER_INSTANCE *encoder_inst);
 
  typedef const void *OSSL_ENCODER_CONSTRUCT(OSSL_ENCODER_INSTANCE *encoder_inst,
                                             void *construct_data);
@@ -110,8 +116,13 @@ encoder chains.
 OSSL_ENCODER_CTX_set_output_type() sets the ending output type.  This must
 be specified, and determines if a complete encoder chain is available.
 
-OSSL_ENCODER_CTX_num_encoders() gets the number of encoders currently added
-to the context I<ctx>.
+OSSL_ENCODER_CTX_set_output_structure() sets the desired output structure.
+This may be used to determines what encoder implementations may be used.
+Depending on the type of object being encoded, the output structure may
+not be relevant.
+
+OSSL_ENCODER_CTX_get_num_encoders() gets the number of encoders currently
+added to the context I<ctx>.
 
 OSSL_ENCODER_CTX_set_construct() sets the constructor I<construct>.
 
@@ -144,18 +155,28 @@ or NULL to indicate that an error has occured.
 
 These utility functions may be used by a constructor:
 
-OSSL_ENCODER_INSTANCE_encoder() can be used to get the encoder method from a
-encoder instance I<encoder_inst>.
+OSSL_ENCODER_INSTANCE_get_encoder() can be used to get the encoder
+implementation of the encoder instance I<encoder_inst>.
+
+OSSL_ENCODER_INSTANCE_get_encoder_ctx() can be used to get the encoder
+implementation's provider context of the encoder instance I<encoder_inst>.
 
-OSSL_ENCODER_INSTANCE_encoder_ctx() can be used to get the encoder method's
-provider context from a encoder instance I<encoder_inst>.
+OSSL_ENCODER_INSTANCE_get_input_type() can be used to get the input type for
+the encoder implementation of the encoder instance I<encoder_inst>.
+This may be NULL.
 
-OSSL_ENCODER_INSTANCE_input_type() can be used to get the input type for
-encoder method from a encoder instance I<encoder_inst>.  This may be NULL.
+OSSL_ENCODER_INSTANCE_get_output_type() can be used to get the output type
+for the encoder implementation of the encoder instance I<encoder_inst>.
+This will never be NULL.
 
-OSSL_ENCODER_INSTANCE_output_type() can be used to get the output type for
-encoder method from a encoder instance I<encoder_inst>.  This will never be
-NULL.
+OSSL_ENCODER_INSTANCE_get_output_type() can be used to get the output type
+for the encoder implementation of the encoder instance I<encoder_inst>.
+This will never be NULL.
+
+OSSL_ENCODER_INSTANCE_get_output_structure() can be used to get the output
+structure for the encoder implementation of the encoder instance
+I<encoder_inst>.
+This may be NULL.
 
 =head1 RETURN VALUES
 
@@ -169,25 +190,28 @@ OSSL_ENCODER_CTX_set_params() returns 1 if all recognised parameters were
 valid, or 0 if one of them was invalid or caused some other failure in the
 implementation.
 
-OSSL_DECODER_CTX_add_decoder(), OSSL_DECODER_CTX_add_extra(),
-OSSL_DECODER_CTX_set_construct(), OSSL_DECODER_CTX_set_construct_data() and
-OSSL_DECODER_CTX_set_cleanup() return 1 on success, or 0 on failure.
+OSSL_ENCODER_CTX_add_encoder(), OSSL_ENCODER_CTX_add_extra(),
+OSSL_ENCODER_CTX_set_construct(), OSSL_ENCODER_CTX_set_construct_data() and
+OSSL_ENCODER_CTX_set_cleanup() return 1 on success, or 0 on failure.
 
-OSSL_DECODER_CTX_num_decoders() returns the current number of decoders.  It
-returns 0 if I<ctx> is NULL.
+OSSL_ENCODER_CTX_get_num_encoders() returns the current number of encoders.
+It returns 0 if I<ctx> is NULL.
 
-OSSL_DECODER_INSTANCE_decoder() returns an B<OSSL_DECODER> pointer on
+OSSL_ENCODER_INSTANCE_get_encoder() returns an B<OSSL_ENCODER> pointer on
 success, or NULL on failure.
 
-OSSL_DECODER_INSTANCE_decoder_ctx() returns a provider context pointer on
+OSSL_ENCODER_INSTANCE_get_encoder_ctx() returns a provider context pointer on
 success, or NULL on failure.
 
-OSSL_ENCODER_INSTANCE_input_type() returns a string with the name of the
+OSSL_ENCODER_INSTANCE_get_output_type() returns a string with the name of the
 input type, if relevant.  NULL is a valid returned value.
 
-OSSL_ENCODER_INSTANCE_output_type() returns a string with the name of the
+OSSL_ENCODER_INSTANCE_get_output_type() returns a string with the name of the
 output type.
 
+OSSL_ENCODER_INSTANCE_get_output_structure() returns a string with the name
+of the output structure.
+
 =head1 SEE ALSO
 
 L<provider(7)>, L<OSSL_ENCODER(3)>
diff --git a/doc/man7/provider-encoder.pod b/doc/man7/provider-encoder.pod
index 8048458b94..92a8b2d3ec 100644
--- a/doc/man7/provider-encoder.pod
+++ b/doc/man7/provider-encoder.pod
@@ -251,6 +251,19 @@ This parameter is I<mandatory>.
 in a set of properties, it would be possible to determine the output type
 from the C<output> property.
 
+=item "output-structure" (B<OSSL_ENCODER_PARAM_OUTPUT_STRUCTURE>) <UTF8 string>
+
+This is used to specify the outermost output structure for an ENCODER
+implementation.
+
+For example, an output of type "DER" for a key pair could be structured
+using PKCS#8, or a key type specific structure, such as PKCS#1 for RSA
+keys.
+
+=for comment If we had functionality to get the value of a specific property
+in a set of properties, it would be possible to determine the output
+structure from the C<structure> property.
+
 =back
 
 =head2 Encoder operation parameters
diff --git a/include/openssl/core_names.h b/include/openssl/core_names.h
index 850351bd30..f0b0cd0163 100644
--- a/include/openssl/core_names.h
+++ b/include/openssl/core_names.h
@@ -453,10 +453,11 @@ extern "C" {
 /*
  * Encoder / decoder parameters
  */
-#define OSSL_ENCODER_PARAM_CIPHER       OSSL_ALG_PARAM_CIPHER
-#define OSSL_ENCODER_PARAM_PROPERTIES   OSSL_ALG_PARAM_PROPERTIES
-#define OSSL_ENCODER_PARAM_INPUT_TYPE   "input-type"
-#define OSSL_ENCODER_PARAM_OUTPUT_TYPE  "output-type"
+#define OSSL_ENCODER_PARAM_CIPHER           OSSL_ALG_PARAM_CIPHER
+#define OSSL_ENCODER_PARAM_PROPERTIES       OSSL_ALG_PARAM_PROPERTIES
+#define OSSL_ENCODER_PARAM_INPUT_TYPE       "input-type"
+#define OSSL_ENCODER_PARAM_OUTPUT_TYPE      "output-type"
+#define OSSL_ENCODER_PARAM_OUTPUT_STRUCTURE "output-structure"
 
 #define OSSL_DECODER_PARAM_PROPERTIES       OSSL_ALG_PARAM_PROPERTIES
 #define OSSL_DECODER_PARAM_INPUT_TYPE       "input-type"
diff --git a/include/openssl/encoder.h b/include/openssl/encoder.h
index 669f688b2d..760ff05c77 100644
--- a/include/openssl/encoder.h
+++ b/include/openssl/encoder.h
@@ -65,9 +65,11 @@ int OSSL_ENCODER_CTX_set_passphrase_ui(OSSL_ENCODER_CTX *ctx,
 int OSSL_ENCODER_CTX_set_cipher(OSSL_ENCODER_CTX *ctx,
                                 const char *cipher_name,
                                 const char *propquery);
+int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection);
 int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
                                      const char *output_type);
-int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection);
+int OSSL_ENCODER_CTX_set_output_structure(OSSL_ENCODER_CTX *ctx,
+                                          const char *output_structure);
 
 /* Utilities to add encoders */
 int OSSL_ENCODER_CTX_add_encoder(OSSL_ENCODER_CTX *ctx, OSSL_ENCODER *encoder);
@@ -84,6 +86,8 @@ const char *
 OSSL_ENCODER_INSTANCE_get_input_type(OSSL_ENCODER_INSTANCE *encoder_inst);
 const char *
 OSSL_ENCODER_INSTANCE_get_output_type(OSSL_ENCODER_INSTANCE *encoder_inst);
+const char *
+OSSL_ENCODER_INSTANCE_get_output_structure(OSSL_ENCODER_INSTANCE *encoder_inst);
 
 typedef const void *OSSL_ENCODER_CONSTRUCT(OSSL_ENCODER_INSTANCE *encoder_inst,
                                            void *construct_data);
diff --git a/util/libcrypto.num b/util/libcrypto.num
index 40e1fcb43a..5bd793efbc 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -5288,3 +5288,5 @@ EVP_PKEY_get1_encoded_public_key        ?	3_0_0	EXIST::FUNCTION:
 OSSL_DECODER_CTX_set_selection          ?	3_0_0	EXIST::FUNCTION:
 OSSL_DECODER_CTX_set_input_structure    ?	3_0_0	EXIST::FUNCTION:
 OSSL_DECODER_INSTANCE_get_input_structure ?	3_0_0	EXIST::FUNCTION:
+OSSL_ENCODER_CTX_set_output_structure   ?	3_0_0	EXIST::FUNCTION:
+OSSL_ENCODER_INSTANCE_get_output_structure ?	3_0_0	EXIST::FUNCTION: