PROV: Fix bignum printout in text serializers

The common routine ossl_prov_print_labeled_bignum() didn't print the
BIGNUM quite the way it should.  It treated the limbs in a big endian
fashion, when they are really organised in a little endian fashion.

Furthermore, we make it inherit the behaviour from the print of legacy
keys, where a number starting with the high bit set gets an extra zero
printed first.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10891)

2 files changed, 49 insertions, 40 deletions

diff --git a/providers/implementations/serializers/serializer_common.c b/providers/implementations/serializers/serializer_common.c
index 6aa4e64e7a..cc6f17908c 100644
--- a/providers/implementations/serializers/serializer_common.c
+++ b/providers/implementations/serializers/serializer_common.c
@@ -138,29 +138,29 @@ OSSL_OP_keymgmt_importkey_fn *ossl_prov_get_importkey(const OSSL_DISPATCH *fns)
 # endif
 
 int ossl_prov_print_labeled_bignum(BIO *out, const char *label,
-                                   const BIGNUM *n)
+                                   const BIGNUM *bn)
 {
     const char *neg;
     const char *post_label_spc = " ";
     int bytes;
     BN_ULONG *words;
-    int i, off;
+    int n, i;
 
-    if (n == NULL)
+    if (bn == NULL)
         return 0;
     if (label == NULL) {
         label = "";
         post_label_spc = "";
     }
 
-    bytes = BN_num_bytes(n);
-    words = bn_get_words(n);
-    neg = BN_is_negative(n) ? "-" : "";
+    bytes = BN_num_bytes(bn);
+    words = bn_get_words(bn);
+    neg = BN_is_negative(bn) ? "-" : "";
 
-    if (BN_is_zero(n))
+    if (BN_is_zero(bn))
         return ossl_prov_bio_printf(out, "%s%s0\n", label, post_label_spc);
 
-    if (BN_num_bytes(n) <= BN_BYTES)
+    if (BN_num_bytes(bn) <= BN_BYTES)
         return ossl_prov_bio_printf(out,
                                     "%s%s%s" BN_FMTu " (%s0x" BN_FMTx ")\n",
                                     label, post_label_spc, neg, words[0],
@@ -172,44 +172,53 @@ int ossl_prov_print_labeled_bignum(BIO *out, const char *label,
     if (ossl_prov_bio_printf(out, "%s%s\n", label, neg) <= 0)
         return 0;
 
-    /* Skip past the zero bytes in the first word */
-    for (off = 0; off < BN_BYTES; off++) {
-        BN_ULONG l = words[0];
-        int o = 8 * (BN_BYTES - off - 1);
-        int b = ((l & (0xffLU << o)) >> o) & 0xff;
-
-        if (b != 0)
-            break;
-    }
-    /* print 16 hex digits per line, indented with 4 spaces */
-    for (i = 0; i < bytes; i += 16) {
-        int j, step;
-
-        if (ossl_prov_bio_printf(out, "    ") <= 0)
-            return 0;
-
-        for (j = 0; i + j < bytes && j < 16; j += BN_BYTES - step) {
-            int k;
-            BN_ULONG l = words[(i + j + off) / BN_BYTES];
-
-            step = (i + j + off) % BN_BYTES;
-
-            for (k = step; k < BN_BYTES; k++) {
-                int o = 8 * (BN_BYTES - k - 1);
-                int b = ((l & (0xffLU << o)) >> o) & 0xff;
+    /* Keep track of how many bytes we have printed out so far */
+    n = 0;
+
+    /*
+     * OpenSSL BIGNUMs are little endian limbs, so we print them last to
+     * first limb.
+     * i is used as limb index, j is used as the "byte index" in the limb
+     */
+    for (i = bytes / BN_BYTES - 1; i >= 0; i--) {
+        BN_ULONG l = words[i];
+        int  j;
+
+        for (j = BN_BYTES - 1; j >= 0; j--) {
+            int o = 8 * j;
+            int b = ((l & (0xffLU << o)) >> o) & 0xff;
+
+            /* Indent every new line with 4 spaces */
+            if ((n % 15) == 0) {
+                if (n > 0)
+                    if (ossl_prov_bio_printf(out, "\n") <= 0)
+                        return 0;
+                if (ossl_prov_bio_printf(out, "    ") <= 0)
+                    return 0;
+            }
 
-                /* We may have reached the number of bytes prematurely */
-                if (i + j + k - off >= bytes)
-                    break;
+            /*
+             * Upper bit set, then we print an extra zero and pretend the
+             * BIGNUM was one byte longer
+             */
+            if (n == 0 && b > 127) {
+                if (ossl_prov_bio_printf(out, "%02x:", 0) <= 0)
+                    return 0;
+                n++;
+                bytes++;
+            }
 
+            if (++n < bytes) {
                 if (ossl_prov_bio_printf(out, "%02x:", b) <= 0)
                     return 0;
+            } else {
+                if (ossl_prov_bio_printf(out, "%02x", b) <= 0)
+                    return 0;
             }
         }
-
-        if (ossl_prov_bio_printf(out, "\n") <= 0)
-            return 0;
     }
+    if (ossl_prov_bio_printf(out, "\n") <= 0)
+        return 0;
 
     return 1;
 }
diff --git a/providers/implementations/serializers/serializer_local.h b/providers/implementations/serializers/serializer_local.h
index 57365f94a9..8507a740a2 100644
--- a/providers/implementations/serializers/serializer_local.h
+++ b/providers/implementations/serializers/serializer_local.h
@@ -54,7 +54,7 @@ int ossl_prov_dsa_pub_to_der(const void *dsa, unsigned char **pder);
 int ossl_prov_dsa_priv_to_der(const void *dsa, unsigned char **pder);
 
 int ossl_prov_print_labeled_bignum(BIO *out, const char *label,
-                                   const BIGNUM *n);
+                                   const BIGNUM *bn);
 int ossl_prov_print_rsa(BIO *out, RSA *rsa, int priv);
 
 enum dh_print_type {