ENCODER: Refactor the OSSL_ENCODER API to be more like OSSL_DECODER

OSSL_ENCODER was developed before OSSL_DECODER, so the idea of
chaining and the resulting API came later.  This series of changes
brings the same sort of API and functionality back to OSSL_ENCODER,
making the two APIs more consistent with each other.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12873)

3 files changed, 47 insertions, 29 deletions

diff --git a/include/crypto/encoder.h b/include/crypto/encoder.h
index f75a031c32..2f036456a2 100644
--- a/include/crypto/encoder.h
+++ b/include/crypto/encoder.h
@@ -11,5 +11,3 @@
 
 OSSL_ENCODER *ossl_encoder_fetch_by_number(OPENSSL_CTX *libctx, int id,
                                            const char *properties);
-OSSL_DECODER *ossl_decoder_fetch_by_number(OPENSSL_CTX *libctx, int id,
-                                           const char *properties);
diff --git a/include/openssl/encoder.h b/include/openssl/encoder.h
index 10b2bc9188..6698769e24 100644
--- a/include/openssl/encoder.h
+++ b/include/openssl/encoder.h
@@ -42,26 +42,59 @@ void OSSL_ENCODER_do_all_provided(OPENSSL_CTX *libctx,
 void OSSL_ENCODER_names_do_all(const OSSL_ENCODER *encoder,
                                void (*fn)(const char *name, void *data),
                                void *data);
+const OSSL_PARAM *OSSL_ENCODER_gettable_params(OSSL_ENCODER *encoder);
+int OSSL_ENCODER_get_params(OSSL_ENCODER *encoder, OSSL_PARAM params[]);
 
 const OSSL_PARAM *OSSL_ENCODER_settable_ctx_params(OSSL_ENCODER *encoder);
-OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new(OSSL_ENCODER *encoder);
-const OSSL_ENCODER *OSSL_ENCODER_CTX_get_encoder(OSSL_ENCODER_CTX *ctx);
+OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new(void);
 int OSSL_ENCODER_CTX_set_params(OSSL_ENCODER_CTX *ctx,
                                 const OSSL_PARAM params[]);
 void OSSL_ENCODER_CTX_free(OSSL_ENCODER_CTX *ctx);
 
 /* Utilities that help set specific parameters */
-int OSSL_ENCODER_CTX_set_cipher(OSSL_ENCODER_CTX *ctx,
-                                const char *cipher_name,
-                                const char *propquery);
 int OSSL_ENCODER_CTX_set_passphrase(OSSL_ENCODER_CTX *ctx,
-                                    const unsigned char *kstr,
-                                    size_t klen);
+                                    const unsigned char *kstr, size_t klen);
+int OSSL_ENCODER_CTX_set_pem_password_cb(OSSL_ENCODER_CTX *ctx,
+                                         pem_password_cb *cb, void *cbarg);
 int OSSL_ENCODER_CTX_set_passphrase_cb(OSSL_ENCODER_CTX *ctx,
-                                       pem_password_cb *cb, void *cbarg);
+                                       OSSL_PASSPHRASE_CALLBACK *cb,
+                                       void *cbarg);
 int OSSL_ENCODER_CTX_set_passphrase_ui(OSSL_ENCODER_CTX *ctx,
                                        const UI_METHOD *ui_method,
                                        void *ui_data);
+int OSSL_ENCODER_CTX_set_cipher(OSSL_ENCODER_CTX *ctx,
+                                const char *cipher_name,
+                                const char *propquery);
+int OSSL_ENCODER_CTX_set_output_type(OSSL_ENCODER_CTX *ctx,
+                                     const char *output_type);
+int OSSL_ENCODER_CTX_set_selection(OSSL_ENCODER_CTX *ctx, int selection);
+
+/* Utilities to add encoders */
+int OSSL_ENCODER_CTX_add_encoder(OSSL_ENCODER_CTX *ctx, OSSL_ENCODER *encoder);
+int OSSL_ENCODER_CTX_add_extra(OSSL_ENCODER_CTX *ctx,
+                               OPENSSL_CTX *libctx, const char *propq);
+int OSSL_ENCODER_CTX_get_num_encoders(OSSL_ENCODER_CTX *ctx);
+
+typedef struct ossl_encoder_instance_st OSSL_ENCODER_INSTANCE;
+OSSL_ENCODER *
+OSSL_ENCODER_INSTANCE_get_encoder(OSSL_ENCODER_INSTANCE *encoder_inst);
+void *
+OSSL_ENCODER_INSTANCE_get_encoder_ctx(OSSL_ENCODER_INSTANCE *encoder_inst);
+const char *
+OSSL_ENCODER_INSTANCE_get_input_type(OSSL_ENCODER_INSTANCE *encoder_inst);
+const char *
+OSSL_ENCODER_INSTANCE_get_output_type(OSSL_ENCODER_INSTANCE *encoder_inst);
+
+typedef const void *OSSL_ENCODER_CONSTRUCT(OSSL_ENCODER_INSTANCE *encoder_inst,
+                                           void *construct_data);
+typedef void OSSL_ENCODER_CLEANUP(void *construct_data);
+
+int OSSL_ENCODER_CTX_set_construct(OSSL_ENCODER_CTX *ctx,
+                                   OSSL_ENCODER_CONSTRUCT *construct);
+int OSSL_ENCODER_CTX_set_construct_data(OSSL_ENCODER_CTX *ctx,
+                                        void *construct_data);
+int OSSL_ENCODER_CTX_set_cleanup(OSSL_ENCODER_CTX *ctx,
+                                 OSSL_ENCODER_CLEANUP *cleanup);
 
 /* Utilities to output the object to encode */
 int OSSL_ENCODER_to_bio(OSSL_ENCODER_CTX *ctx, BIO *out);
@@ -75,25 +108,11 @@ int OSSL_ENCODER_to_fp(OSSL_ENCODER_CTX *ctx, FILE *fp);
  * This is more useful than calling OSSL_ENCODER_CTX_new().
  */
 OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new_by_EVP_PKEY(const EVP_PKEY *pkey,
+                                                   const char *output_type,
+                                                   int selection,
+                                                   OPENSSL_CTX *libctx,
                                                    const char *propquery);
 
-/*
- * These macros define the last argument to pass to
- * OSSL_ENCODER_CTX_new_by_TYPE().
- */
-# define OSSL_ENCODER_PUBKEY_TO_PEM_PQ "format=pem,type=public"
-# define OSSL_ENCODER_PrivateKey_TO_PEM_PQ "format=pem,type=private"
-# define OSSL_ENCODER_Parameters_TO_PEM_PQ "format=pem,type=parameters"
-
-# define OSSL_ENCODER_PUBKEY_TO_DER_PQ "format=der,type=public"
-# define OSSL_ENCODER_PrivateKey_TO_DER_PQ "format=der,type=private"
-# define OSSL_ENCODER_Parameters_TO_DER_PQ "format=der,type=parameters"
-
-/* Corresponding macros for text output */
-# define OSSL_ENCODER_PUBKEY_TO_TEXT_PQ "format=text,type=public"
-# define OSSL_ENCODER_PrivateKey_TO_TEXT_PQ "format=text,type=private"
-# define OSSL_ENCODER_Parameters_TO_TEXT_PQ "format=text,type=parameters"
-
 # ifdef __cplusplus
 }
 # endif
diff --git a/include/openssl/encodererr.h b/include/openssl/encodererr.h
index 007070e0c0..e146d6ec92 100644
--- a/include/openssl/encodererr.h
+++ b/include/openssl/encodererr.h
@@ -30,7 +30,8 @@ int ERR_load_OSSL_ENCODER_strings(void);
 /*
  * OSSL_ENCODER reason codes.
  */
-# define OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY       100
-# define OSSL_ENCODER_R_ENCODER_NOT_FOUND           101
+# define OSSL_ENCODER_R_ENCODER_NOT_FOUND                 101
+# define OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY          100
+# define OSSL_ENCODER_R_MISSING_GET_PARAMS                102
 
 #endif