ENCODER: Refactor provider implementations, and some cleanup

The encoder implementations were implemented by unnecessarily copying
code into numerous topical source files, making them hard to maintain.
This changes merges all those into two source files, one that encodes
into DER and PEM, the other to text.

Diverse small cleanups are included.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12803)

1 files changed, 63 insertions, 5 deletions

diff --git a/providers/implementations/encode_decode/decode_der2key.c b/providers/implementations/encode_decode/decode_der2key.c
index b8b268217d..011f05803d 100644
--- a/providers/implementations/encode_decode/decode_der2key.c
+++ b/providers/implementations/encode_decode/decode_der2key.c
@@ -17,13 +17,71 @@
 #include <openssl/core_names.h>
 #include <openssl/core_object.h>
 #include <openssl/crypto.h>
+#include <openssl/err.h>
 #include <openssl/params.h>
+#include <openssl/pem.h>         /* PEM_BUFSIZE and public PEM functions */
+#include <openssl/pkcs12.h>
 #include <openssl/x509.h>
+#include "internal/cryptlib.h"   /* ossl_assert() */
+#include "internal/asn1.h"
+#include "crypto/ecx.h"
 #include "prov/bio.h"
 #include "prov/implementations.h"
-#include "encoder_local.h"
+#include "prov/providercommonerr.h"
+#include "endecoder_local.h"
 
-static OSSL_FUNC_decoder_newctx_fn der2rsa_newctx;
+static int read_der(PROV_CTX *provctx, OSSL_CORE_BIO *cin,
+                    unsigned char **data, long *len)
+{
+    BUF_MEM *mem = NULL;
+    BIO *in = bio_new_from_core_bio(provctx, cin);
+    int ok = (asn1_d2i_read_bio(in, &mem) >= 0);
+
+    if (ok) {
+        *data = (unsigned char *)mem->data;
+        *len = (long)mem->length;
+        OPENSSL_free(mem);
+    }
+    BIO_free(in);
+    return ok;
+}
+
+static int der_from_p8(unsigned char **new_der, long *new_der_len,
+                       unsigned char *input_der, long input_der_len,
+                       OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg)
+{
+    const unsigned char *derp;
+    X509_SIG *p8 = NULL;
+    int ok = 0;
+
+    if (!ossl_assert(new_der != NULL && *new_der == NULL)
+        || !ossl_assert(new_der_len != NULL))
+        return 0;
+
+    derp = input_der;
+    if ((p8 = d2i_X509_SIG(NULL, &derp, input_der_len)) != NULL) {
+        char pbuf[PEM_BUFSIZE];
+        size_t plen = 0;
+
+        if (!pw_cb(pbuf, sizeof(pbuf), &plen, NULL, pw_cbarg)) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_READ_KEY);
+        } else {
+            const X509_ALGOR *alg = NULL;
+            const ASN1_OCTET_STRING *oct = NULL;
+            int len = 0;
+
+            X509_SIG_get0(p8, &alg, &oct);
+            if (PKCS12_pbe_crypt(alg, pbuf, plen, oct->data, oct->length,
+                                 new_der, &len, 0) != NULL)
+                ok = 1;
+            *new_der_len = len;
+        }
+    }
+    X509_SIG_free(p8);
+    return ok;
+}
+
+/* ---------------------------------------------------------------------- */
 
 static OSSL_FUNC_decoder_freectx_fn der2key_freectx;
 static OSSL_FUNC_decoder_gettable_params_fn der2key_gettable_params;
@@ -109,15 +167,14 @@ static int der2key_decode(void *vctx, OSSL_CORE_BIO *cin,
     void *key = NULL;
     int ok = 0;
 
-    if (!ossl_prov_read_der(ctx->provctx, cin, &der, &der_len))
+    if (!read_der(ctx->provctx, cin, &der, &der_len))
         return 0;
 
     /*
      * Opportunistic attempt to decrypt.  If it doesn't work, we try to
      * decode our input unencrypted.
      */
-    if (ossl_prov_der_from_p8(&new_der, &new_der_len, der, der_len,
-                              pw_cb, pw_cbarg)) {
+    if (der_from_p8(&new_der, &new_der_len, der, der_len, pw_cb, pw_cbarg)) {
         OPENSSL_free(der);
         der = new_der;
         der_len = new_der_len;
@@ -203,6 +260,7 @@ static int der2key_export_object(void *vctx,
         { EVP_PKEY_##KEYTYPE, KEYTYPEstr, keytype##_keymgmt_functions,  \
           (extract_key_fn *)extract,                                    \
           (free_key_fn *)free };                                        \
+    static OSSL_FUNC_decoder_newctx_fn der2##keytype##_newctx;          \
     static void *der2##keytype##_newctx(void *provctx)                  \
     {                                                                   \
         return der2key_newctx(provctx, &keytype##_desc);                \