trace: don't leak the line prefix

The openssl app registers trace callbacks which automatically
set a line prefix in the OSSL_TRACE_CTRL_BEGIN callback.
This prefix needs to be cleared in the OSSL_TRACE_CTRL_END
callback, otherwise a memory leak is reported when openssl
is built with crypto-mdebug enabled.

This leak causes the tests to fail when tracing and memory
debugging are enabled.

The leak can be observed by any command that produces trace
output, e.g. by

  OPENSSL_TRACE=ANY util/shlib_wrap.sh  apps/openssl version
  ...
  [00:19:14]  4061 file=apps/bf_prefix.c, line=152, ...
  26 bytes leaked in 1 chunks

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8463)

1 files changed, 16 insertions, 20 deletions

diff --git a/apps/openssl.c b/apps/openssl.c
index 72a0bb0085..db1dbb767d 100644
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -126,30 +126,17 @@ typedef struct tracedata_st {
 static size_t internal_trace_cb(const char *buf, size_t cnt,
                                 int category, int cmd, void *vdata)
 {
-    int ret;
+    int ret = 0;
     tracedata *trace_data = vdata;
-    int set_prefix = 0;
+    union {
+        CRYPTO_THREAD_ID tid;
+        unsigned long ltid;
+    } tid;
+    char buffer[256];
 
     switch (cmd) {
     case OSSL_TRACE_CTRL_BEGIN:
         trace_data->ingroup = 1;
-        set_prefix = 1;
-        break;
-    case OSSL_TRACE_CTRL_DURING:
-        if (!trace_data->ingroup)
-            set_prefix = 1;
-        break;
-    case OSSL_TRACE_CTRL_END:
-        trace_data->ingroup = 0;
-        break;
-    }
-
-    if (set_prefix) {
-        union {
-            CRYPTO_THREAD_ID tid;
-            unsigned long ltid;
-        } tid;
-        char buffer[256];
 
         tid.ltid = 0;
         tid.tid = CRYPTO_THREAD_get_current_id();
@@ -158,8 +145,17 @@ static size_t internal_trace_cb(const char *buf, size_t cnt,
                      OSSL_trace_get_category_name(category));
         BIO_ctrl(trace_data->bio, PREFIX_CTRL_SET_PREFIX,
                  strlen(buffer), buffer);
+        break;
+    case OSSL_TRACE_CTRL_WRITE:
+        ret = BIO_write(trace_data->bio, buf, cnt);
+        break;
+    case OSSL_TRACE_CTRL_END:
+        trace_data->ingroup = 0;
+
+        BIO_ctrl(trace_data->bio, PREFIX_CTRL_SET_PREFIX, 0, NULL);
+
+        break;
     }
-    ret = BIO_write(trace_data->bio, buf, cnt);
 
     return ret < 0 ? 0 : ret;
 }