DECODER: Add tracing

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13248)

3 files changed, 39 insertions, 2 deletions

diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index 64e6bcc185..8e9af13bbb 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -14,6 +14,7 @@
 #include <openssl/evperr.h>
 #include <openssl/ecerr.h>
 #include <openssl/x509err.h>
+#include <openssl/trace.h>
 #include "internal/passphrase.h"
 #include "crypto/decoder.h"
 #include "encoder_local.h"
@@ -210,6 +211,8 @@ void ossl_decoder_instance_free(OSSL_DECODER_INSTANCE *decoder_inst)
 int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx,
                                       OSSL_DECODER_INSTANCE *di)
 {
+    int ok;
+
     if (ctx->decoder_insts == NULL
         && (ctx->decoder_insts =
             sk_OSSL_DECODER_INSTANCE_new_null()) == NULL) {
@@ -217,7 +220,18 @@ int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx,
         return 0;
     }
 
-    return (sk_OSSL_DECODER_INSTANCE_push(ctx->decoder_insts, di) > 0);
+    ok = (sk_OSSL_DECODER_INSTANCE_push(ctx->decoder_insts, di) > 0);
+    if (ok) {
+        OSSL_TRACE_BEGIN(DECODER) {
+            BIO_printf(trc_out,
+                       "(ctx %p) Added decoder instance %p (decoder %p) with:\n",
+                       (void *)ctx, (void *)di, (void *)di->decoder);
+            BIO_printf(trc_out,
+                       "    input type: %s, input structure: %s\n",
+                       di->input_type, di->input_structure);
+        } OSSL_TRACE_END(DECODER);
+    }
+    return ok;
 }
 
 int OSSL_DECODER_CTX_add_decoder(OSSL_DECODER_CTX *ctx, OSSL_DECODER *decoder)
@@ -615,6 +629,13 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
                                  decoder_process, &new_data,
                                  ossl_pw_passphrase_callback_dec,
                                  &new_data.ctx->pwdata);
+
+        OSSL_TRACE_BEGIN(DECODER) {
+            BIO_printf(trc_out,
+                       "(ctx %p) Running decoder instance %p => %d\n",
+                       (void *)new_data.ctx, (void *)new_decoder_inst, ok);
+        } OSSL_TRACE_END(DECODER);
+
         if (ok)
             break;
         err = ERR_peek_last_error();
diff --git a/crypto/encode_decode/decoder_pkey.c b/crypto/encode_decode/decoder_pkey.c
index 1835106b95..3a765c5986 100644
--- a/crypto/encode_decode/decoder_pkey.c
+++ b/crypto/encode_decode/decoder_pkey.c
@@ -14,6 +14,7 @@
 #include <openssl/ui.h>
 #include <openssl/decoder.h>
 #include <openssl/safestack.h>
+#include <openssl/trace.h>
 #include "crypto/evp.h"
 #include "crypto/decoder.h"
 #include "encoder_local.h"
@@ -388,13 +389,27 @@ OSSL_DECODER_CTX_new_by_EVP_PKEY(EVP_PKEY **pkey,
         ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
+
+    OSSL_TRACE_BEGIN(DECODER) {
+        BIO_printf(trc_out,
+                   "(ctx %p) Looking for %s decoders with selection %d\n",
+                   (void *)ctx, keytype, selection);
+        BIO_printf(trc_out, "    input type: %s, input structure: %s\n",
+                   input_type, input_structure);
+    } OSSL_TRACE_END(DECODER);
+
     if (OSSL_DECODER_CTX_set_input_type(ctx, input_type)
         && OSSL_DECODER_CTX_set_input_structure(ctx, input_structure)
         && OSSL_DECODER_CTX_set_selection(ctx, selection)
         && ossl_decoder_ctx_setup_for_EVP_PKEY(ctx, pkey, keytype,
                                                libctx, propquery)
-        && OSSL_DECODER_CTX_add_extra(ctx, libctx, propquery))
+        && OSSL_DECODER_CTX_add_extra(ctx, libctx, propquery)) {
+        OSSL_TRACE_BEGIN(DECODER) {
+            BIO_printf(trc_out, "(ctx %p) Got %d decoders\n",
+                       (void *)ctx, OSSL_DECODER_CTX_get_num_decoders(ctx));
+        } OSSL_TRACE_END(DECODER);
         return ctx;
+    }
 
     OSSL_DECODER_CTX_free(ctx);
     return NULL;
diff --git a/crypto/trace.c b/crypto/trace.c
index a8316c12b1..46a1800753 100644
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -136,6 +136,7 @@ static const struct trace_category_st trace_categories[] = {
     TRACE_CATEGORY_(X509V3_POLICY),
     TRACE_CATEGORY_(BN_CTX),
     TRACE_CATEGORY_(STORE),
+    TRACE_CATEGORY_(DECODER),
 };
 
 const char *OSSL_trace_get_category_name(int num)