Fix ERR_print_errors so that it matches the documented format in doc/man3/ERR_error_string.pod

Fixes #11743

The ouput format had 2 issues that caused it not to match the expected documented format:
(1) At some point the thread id printing was changed to use the OPENSSL_hex2str method which puts ':' between hex bytes.
    An internal function that skips the seperator has been added.
(2) The error code no longer exists. So this was completely removed from the string. It is now replaced by ::

As an example:
  00:77:6E:52:14:7F:00:00:error:asn1 encoding routines:asn1_check_tlen:wrong tag:crypto/asn1/tasn_dec.c:1135:
Is now:
  00776E52147F0000:error::asn1 encoding routines:asn1_check_tlen:wrong tag:crypto/asn1/tasn_dec.c:1135:

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11789)

4 files changed, 198 insertions, 1 deletions

diff --git a/test/build.info b/test/build.info
index 112b68c22f..9697e55f12 100644
--- a/test/build.info
+++ b/test/build.info
@@ -54,7 +54,7 @@ IF[{- !$disabled{tests} -}]
           http_test servername_test ocspapitest fatalerrtest tls13ccstest \
           sysdefaulttest errtest ssl_ctx_test gosttest \
           context_internal_test aesgcmtest params_test evp_pkey_dparams_test \
-          keymgmt_internal_test
+          keymgmt_internal_test hexstr_test
 
   SOURCE[confdump]=confdump.c
   INCLUDE[confdump]=../include ../apps/include
@@ -736,6 +736,11 @@ IF[{- !$disabled{tests} -}]
   INCLUDE[params_test]=.. ../include ../apps/include
   DEPEND[params_test]=../libcrypto.a libtestutil.a
 
+  PROGRAMS{noinst}=hexstr_test
+  SOURCE[hexstr_test]=hexstr_test.c
+  INCLUDE[hexstr_test]=.. ../include ../apps/include
+  DEPEND[hexstr_test]=../libcrypto.a libtestutil.a
+
   PROGRAMS{noinst}=namemap_internal_test
   SOURCE[namemap_internal_test]=namemap_internal_test.c
   INCLUDE[namemap_internal_test]=.. ../include ../apps/include
diff --git a/test/errtest.c b/test/errtest.c
index 179c338c45..cc2f6612d1 100644
--- a/test/errtest.c
+++ b/test/errtest.c
@@ -7,6 +7,7 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <string.h>
 #include <openssl/opensslconf.h>
 #include <openssl/err.h>
 
@@ -18,6 +19,46 @@
 # include <errno.h>
 #endif
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+# define IS_HEX(ch) ((ch >= '0' && ch <='9') || (ch >= 'A' && ch <='F'))
+
+static int test_print_error_format(void)
+{
+    static const char expected[] =
+        ":error::system library:test_print_error_format:Operation not permitted:"
+# ifndef OPENSSL_NO_FILENAMES
+        "errtest.c:30:";
+# else
+        ":0:";
+# endif
+    char *out = NULL, *p = NULL;
+    int ret = 0, len;
+    BIO *bio = NULL;
+
+    if (!TEST_ptr(bio = BIO_new(BIO_s_mem())))
+        return 0;
+
+    ERR_PUT_error(ERR_LIB_SYS, 0, 1, "errtest.c", 30);
+    ERR_print_errors(bio);
+
+    if (!TEST_int_gt(len = BIO_get_mem_data(bio, &out), 0))
+        goto err;
+    /* Skip over the variable thread id at the start of the string */
+    for (p = out; *p != ':' && *p != 0; ++p) {
+        if (!TEST_true(IS_HEX(*p)))
+            goto err;
+    }
+    if (!TEST_true(*p != 0)
+        || !TEST_strn_eq(expected, p, strlen(expected)))
+        goto err;
+
+    ret = 1;
+err:
+    BIO_free(bio);
+    return ret;
+}
+#endif
+
 /* Test that querying the error queue preserves the OS error. */
 static int preserves_system_error(void)
 {
@@ -79,5 +120,8 @@ int setup_tests(void)
     ADD_TEST(preserves_system_error);
     ADD_TEST(vdata_appends);
     ADD_TEST(raised_error);
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    ADD_TEST(test_print_error_format);
+#endif
     return 1;
 }
diff --git a/test/hexstr_test.c b/test/hexstr_test.c
new file mode 100644
index 0000000000..c4f13b6d53
--- /dev/null
+++ b/test/hexstr_test.c
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+/*
+ * This program tests the use of OSSL_PARAM, currently in raw form.
+ */
+
+#include "internal/nelem.h"
+#include "internal/cryptlib.h"
+#include "testutil.h"
+
+struct testdata
+{
+    const char *in;
+    const unsigned char *expected;
+    size_t expected_len;
+    const char sep;
+};
+
+static const unsigned char test_1[] = { 0xAB, 0xCD, 0xEF, 0xF1 };
+static const unsigned char test_2[] = { 0xAB, 0xCD, 0xEF, 0x76, 0x00 };
+
+static struct testdata tbl_testdata[] = {
+    {
+        "AB:CD:EF:F1",
+        test_1, sizeof(test_1),
+        ':',
+    },
+    {
+        "AB:CD:EF:76:00",
+        test_2, sizeof(test_2),
+        ':',
+    },
+    {
+        "AB_CD_EF_F1",
+        test_1, sizeof(test_1),
+        '_',
+    },
+    {
+        "AB_CD_EF_76_00",
+        test_2, sizeof(test_2),
+        '_',
+    },
+    {
+        "ABCDEFF1",
+        test_1, sizeof(test_1),
+        '\0',
+    },
+    {
+        "ABCDEF7600",
+        test_2, sizeof(test_2),
+        '\0',
+    },
+};
+
+static int test_hexstr_sep_to_from(int test_index)
+{
+    int ret = 0;
+    long len = 0;
+    unsigned char *buf = NULL;
+    char *out = NULL;
+    struct testdata *test = &tbl_testdata[test_index];
+
+    if (!TEST_ptr(buf = openssl_hexstr2buf_sep(test->in, &len, test->sep))
+        || !TEST_mem_eq(buf, len, test->expected, test->expected_len)
+        || !TEST_ptr(out = openssl_buf2hexstr_sep(buf, len, test->sep))
+        || !TEST_str_eq(out, test->in))
+       goto err;
+
+    ret = 1;
+err:
+    OPENSSL_free(buf);
+    OPENSSL_free(out);
+    return ret;
+}
+
+static int test_hexstr_to_from(int test_index)
+{
+    int ret = 0;
+    long len = 0;
+    unsigned char *buf = NULL;
+    char *out = NULL;
+    struct testdata *test = &tbl_testdata[test_index];
+
+    if (test->sep != '_') {
+        if (!TEST_ptr(buf = OPENSSL_hexstr2buf(test->in, &len))
+            || !TEST_mem_eq(buf, len, test->expected, test->expected_len)
+            || !TEST_ptr(out = OPENSSL_buf2hexstr(buf, len)))
+           goto err;
+        if (test->sep == ':') {
+            if (!TEST_str_eq(out, test->in))
+                goto err;
+        } else if (!TEST_str_ne(out, test->in)) {
+            goto err;
+        }
+    } else {
+        if (!TEST_ptr_null(buf = OPENSSL_hexstr2buf(test->in, &len)))
+            goto err;
+    }
+    ret = 1;
+err:
+    OPENSSL_free(buf);
+    OPENSSL_free(out);
+    return ret;
+}
+
+static int test_hexstr_ex_to_from(int test_index)
+{
+    size_t len = 0;
+    char out[64];
+    unsigned char buf[64];
+    struct testdata *test = &tbl_testdata[test_index];
+
+    return TEST_true(OPENSSL_hexstr2buf_ex(buf, sizeof(buf), &len, test->in))
+           && TEST_mem_eq(buf, len, test->expected, test->expected_len)
+           && TEST_true(OPENSSL_buf2hexstr_ex(out, sizeof(out), NULL, buf, len))
+           && TEST_str_eq(out, test->in);
+}
+
+int setup_tests(void)
+{
+    ADD_ALL_TESTS(test_hexstr_sep_to_from, OSSL_NELEM(tbl_testdata));
+    ADD_ALL_TESTS(test_hexstr_to_from, OSSL_NELEM(tbl_testdata));
+    ADD_ALL_TESTS(test_hexstr_ex_to_from, 2);
+    return 1;
+}
diff --git a/test/recipes/04-test_hexstring.t b/test/recipes/04-test_hexstring.t
new file mode 100644
index 0000000000..664868fe60
--- /dev/null
+++ b/test/recipes/04-test_hexstring.t
@@ -0,0 +1,15 @@
+#! /usr/bin/env perl
+# Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use OpenSSL::Test;
+use OpenSSL::Test::Simple;
+
+setup("test_hexstring");
+
+simple_test("test_hexstring", "hexstr_test");