Misc BN fixes

Never output -0; make "negative zero" an impossibility.
Do better checking on BN_rand top/bottom requirements and #bits.
Update doc.
Ignoring trailing garbage in BN_asc2bn.

Port this commit from boringSSL: https://boringssl.googlesource.com/boringssl/+/899b9b19a4cd3fe526aaf5047ab9234cdca19f7d%5E!/
        Ensure |BN_div| never gives negative zero in the no_branch code.

        Have |bn_correct_top| fix |bn->neg| if the input is zero so that we
        don't have negative zeros lying around.

        Thanks to Brian Smith for noticing.

Reviewed-by: Richard Levitte <levitte@openssl.org>

1 files changed, 13 insertions, 10 deletions

diff --git a/crypto/bn/bn_print.c b/crypto/bn/bn_print.c
index 39fb03412e..a16bde862c 100644
--- a/crypto/bn/bn_print.c
+++ b/crypto/bn/bn_print.c
@@ -23,12 +23,9 @@ char *BN_bn2hex(const BIGNUM *a)
     char *buf;
     char *p;
 
-    if (a->neg && BN_is_zero(a)) {
-        /* "-0" == 3 bytes including NULL terminator */
-        buf = OPENSSL_malloc(3);
-    } else {
-        buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
-    }
+    if (BN_is_zero(a))
+        return OPENSSL_strdup("0");
+    buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
     if (buf == NULL) {
         BNerr(BN_F_BN_BN2HEX, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -186,10 +183,12 @@ int BN_hex2bn(BIGNUM **bn, const char *a)
     }
     ret->top = h;
     bn_correct_top(ret);
-    ret->neg = neg;
 
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -241,7 +240,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
     if (j == BN_DEC_NUM)
         j = 0;
     l = 0;
-    while (*a) {
+    while (--i >= 0) {
         l *= 10;
         l += *a - '0';
         a++;
@@ -253,11 +252,13 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
             j = 0;
         }
     }
-    ret->neg = neg;
 
     bn_correct_top(ret);
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -268,6 +269,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
 int BN_asc2bn(BIGNUM **bn, const char *a)
 {
     const char *p = a;
+
     if (*p == '-')
         p++;
 
@@ -278,7 +280,8 @@ int BN_asc2bn(BIGNUM **bn, const char *a)
         if (!BN_dec2bn(bn, p))
             return 0;
     }
-    if (*a == '-')
+    /* Don't set the negative flag if it's zero. */
+    if (*a == '-' && (*bn)->top != 0)
         (*bn)->neg = 1;
     return 1;
 }