Fix ERR_print_errors so that it matches the documented format in doc/man3/ERR_error_string.pod

Fixes #11743

The ouput format had 2 issues that caused it not to match the expected documented format:
(1) At some point the thread id printing was changed to use the OPENSSL_hex2str method which puts ':' between hex bytes.
    An internal function that skips the seperator has been added.
(2) The error code no longer exists. So this was completely removed from the string. It is now replaced by ::

As an example:
  00:77:6E:52:14:7F:00:00:error:asn1 encoding routines:asn1_check_tlen:wrong tag:crypto/asn1/tasn_dec.c:1135:
Is now:
  00776E52147F0000:error::asn1 encoding routines:asn1_check_tlen:wrong tag:crypto/asn1/tasn_dec.c:1135:

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11789)

1 files changed, 44 insertions, 0 deletions

diff --git a/test/errtest.c b/test/errtest.c
index 179c338c45..cc2f6612d1 100644
--- a/test/errtest.c
+++ b/test/errtest.c
@@ -7,6 +7,7 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <string.h>
 #include <openssl/opensslconf.h>
 #include <openssl/err.h>
 
@@ -18,6 +19,46 @@
 # include <errno.h>
 #endif
 
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+# define IS_HEX(ch) ((ch >= '0' && ch <='9') || (ch >= 'A' && ch <='F'))
+
+static int test_print_error_format(void)
+{
+    static const char expected[] =
+        ":error::system library:test_print_error_format:Operation not permitted:"
+# ifndef OPENSSL_NO_FILENAMES
+        "errtest.c:30:";
+# else
+        ":0:";
+# endif
+    char *out = NULL, *p = NULL;
+    int ret = 0, len;
+    BIO *bio = NULL;
+
+    if (!TEST_ptr(bio = BIO_new(BIO_s_mem())))
+        return 0;
+
+    ERR_PUT_error(ERR_LIB_SYS, 0, 1, "errtest.c", 30);
+    ERR_print_errors(bio);
+
+    if (!TEST_int_gt(len = BIO_get_mem_data(bio, &out), 0))
+        goto err;
+    /* Skip over the variable thread id at the start of the string */
+    for (p = out; *p != ':' && *p != 0; ++p) {
+        if (!TEST_true(IS_HEX(*p)))
+            goto err;
+    }
+    if (!TEST_true(*p != 0)
+        || !TEST_strn_eq(expected, p, strlen(expected)))
+        goto err;
+
+    ret = 1;
+err:
+    BIO_free(bio);
+    return ret;
+}
+#endif
+
 /* Test that querying the error queue preserves the OS error. */
 static int preserves_system_error(void)
 {
@@ -79,5 +120,8 @@ int setup_tests(void)
     ADD_TEST(preserves_system_error);
     ADD_TEST(vdata_appends);
     ADD_TEST(raised_error);
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    ADD_TEST(test_print_error_format);
+#endif
     return 1;
 }