trace: apps/openssl: print the correct category name

Previously, if the openssl application was run with OPENSSL_TRACE=any,
all trace output would just show 'ANY' as the category name, which was
not very useful. To get the correct category name printed in the trace
output, the openssl application now registers separate channels for
each category.

The trace API is unchanged, it is still possible for an application to
register a single channel for the 'ANY' category to see all outputt,
if it does not need this level of detail.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8552)

4 files changed, 46 insertions, 20 deletions

diff --git a/apps/openssl.c b/apps/openssl.c
index 44dbda7625..a63f82b15b 100644
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -185,6 +185,33 @@ static void cleanup_trace(void)
     sk_tracedata_pop_free(trace_data_stack, tracedata_free);
 }
 
+static void setup_trace_category(int category)
+{
+    BIO *channel;
+    tracedata *trace_data;
+
+    if (OSSL_trace_enabled(category))
+        return;
+
+    channel = BIO_push(BIO_new(apps_bf_prefix()),
+                       dup_bio_err(FORMAT_TEXT));
+    trace_data = OPENSSL_zalloc(sizeof(*trace_data));
+
+    if (trace_data == NULL
+        || (trace_data->bio = channel) == NULL
+        || OSSL_trace_set_callback(category, internal_trace_cb,
+                                   trace_data) == 0
+        || sk_tracedata_push(trace_data_stack, trace_data) == 0) {
+
+        fprintf(stderr,
+                "warning: unable to setup trace callback for category '%s'.\n",
+                OSSL_trace_get_category_name(category));
+
+        OSSL_trace_set_callback(category, NULL, NULL);
+        BIO_free_all(channel);
+    }
+}
+
 static void setup_trace(const char *str)
 {
     char *val;
@@ -199,26 +226,15 @@ static void setup_trace(const char *str)
         for (valp = val; (item = strtok(valp, ",")) != NULL; valp = NULL) {
             int category = OSSL_trace_get_category_num(item);
 
-            if (category >= 0) {
-                BIO *channel = BIO_push(BIO_new(apps_bf_prefix()),
-                                        dup_bio_err(FORMAT_TEXT));
-                tracedata *trace_data = OPENSSL_zalloc(sizeof(*trace_data));
-
-                if (trace_data == NULL
-                    || (trace_data->bio = channel) == NULL
-                    || OSSL_trace_set_callback(category, internal_trace_cb,
-                                               trace_data) == 0
-                    || sk_tracedata_push(trace_data_stack, trace_data) == 0) {
-                    OSSL_trace_set_callback(category, NULL, NULL);
-                    BIO_free_all(channel);
-                    fprintf(stderr,
-                            "warning: unable to setup trace callback for category '%s'.\n",
-                            item);
-                }
+            if (category == OSSL_TRACE_CATEGORY_ANY) {
+                while (++category < OSSL_TRACE_CATEGORY_NUM)
+                    setup_trace_category(category);
+                break;
+            } else if (category > 0) {
+                setup_trace_category(category);
             } else {
                 fprintf(stderr,
-                        "warning: unknown trace category: '%s'.\n",
-                        item);
+                        "warning: unknown trace category: '%s'.\n", item);
             }
         }
     }
diff --git a/crypto/trace.c b/crypto/trace.c
index 5e2fec299a..70e93c292d 100644
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -330,7 +330,7 @@ int OSSL_trace_set_channel(int category, BIO *channel)
 #ifndef OPENSSL_NO_TRACE
     if (category >= 0 && category < OSSL_TRACE_CATEGORY_NUM)
         return set_trace_data(category, SIMPLE_CHANNEL, &channel, NULL, NULL,
-                              trace_attach_cb, trace_detach_cb))
+                              trace_attach_cb, trace_detach_cb);
 #endif
     return 0;
 }
diff --git a/doc/man3/OSSL_trace_set_channel.pod b/doc/man3/OSSL_trace_set_channel.pod
index 773a6b1064..0fb7d06a6b 100644
--- a/doc/man3/OSSL_trace_set_channel.pod
+++ b/doc/man3/OSSL_trace_set_channel.pod
@@ -181,6 +181,12 @@ Traces BIGNUM context operations.
 There is also C<OSSL_TRACE_CATEGORY_ANY>, which works as a fallback
 and can be used to get I<all> trace output.
 
+Note, however, that in this case all trace output will effectively be
+associated with the 'ALL' category, which is undesirable if the
+application intends to include the category name in the trace output.
+In this case it is better to register separate channels for each
+trace category instead.
+
 =head1 RETURN VALUES
 
 OSSL_trace_set_channel(), OSSL_trace_set_prefix(),
diff --git a/include/openssl/trace.h b/include/openssl/trace.h
index 767b19fa7d..13cd2dd6f3 100644
--- a/include/openssl/trace.h
+++ b/include/openssl/trace.h
@@ -30,7 +30,11 @@ extern "C" {
  * BIO which sends all trace output it receives to the registered application
  * callback.
  *
- * The ANY category is used as a fallback category.
+ * The ANY category can be used as a fallback category to register a single
+ * channel which receives the output from all categories. However, if the
+ * application intends to print the trace channel name in the line prefix,
+ * it is better to register channels for all categories separately.
+ * (This is how the openssl application does it.)
  */
 # define OSSL_TRACE_CATEGORY_ANY                 0 /* The fallback */
 # define OSSL_TRACE_CATEGORY_TRACE               1