bn/bn_lib.c: add BN_FLG_FIXED_TOP flag.

The new flag marks vectors that were not treated with bn_correct_top,
in other words such vectors are permitted to be zero padded. For now
it's BN_DEBUG-only flag, as initial use case for zero-padded vectors
would be controlled Montgomery multiplication/exponentiation, not
general purpose. For general purpose use another type might be more
appropriate. Advantage of this suggestion is that it's possible to
back-port it...

bn/bn_div.c: fix memory sanitizer problem.
bn/bn_sqr.c: harmonize with BN_mul.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: David Benjamin <davidben@google.com>
(Merged from https://github.com/openssl/openssl/pull/6662)

1 files changed, 11 insertions, 4 deletions

diff --git a/crypto/bn/bn_lib.c b/crypto/bn/bn_lib.c
index 6e1b902d87..b42df82f8e 100644
--- a/crypto/bn/bn_lib.c
+++ b/crypto/bn/bn_lib.c
@@ -289,15 +289,17 @@ BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
     if (b->top > 0)
         memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);
 
-    a->top = b->top;
     a->neg = b->neg;
+    a->top = b->top;
+    a->flags |= b->flags & BN_FLG_FIXED_TOP;
     bn_check_top(a);
     return a;
 }
 
 #define FLAGS_DATA(flags) ((flags) & (BN_FLG_STATIC_DATA \
                                     | BN_FLG_CONSTTIME   \
-                                    | BN_FLG_SECURE))
+                                    | BN_FLG_SECURE      \
+                                    | BN_FLG_FIXED_TOP))
 #define FLAGS_STRUCT(flags) ((flags) & (BN_FLG_MALLOCED))
 
 void BN_swap(BIGNUM *a, BIGNUM *b)
@@ -338,8 +340,9 @@ void BN_clear(BIGNUM *a)
     bn_check_top(a);
     if (a->d != NULL)
         OPENSSL_cleanse(a->d, sizeof(*a->d) * a->dmax);
-    a->top = 0;
     a->neg = 0;
+    a->top = 0;
+    a->flags &= ~BN_FLG_FIXED_TOP;
 }
 
 BN_ULONG BN_get_word(const BIGNUM *a)
@@ -360,6 +363,7 @@ int BN_set_word(BIGNUM *a, BN_ULONG w)
     a->neg = 0;
     a->d[0] = w;
     a->top = (w ? 1 : 0);
+    a->flags &= ~BN_FLG_FIXED_TOP;
     bn_check_top(a);
     return 1;
 }
@@ -596,6 +600,7 @@ int BN_set_bit(BIGNUM *a, int n)
         for (k = a->top; k < i + 1; k++)
             a->d[k] = 0;
         a->top = i + 1;
+        a->flags &= ~BN_FLG_FIXED_TOP;
     }
 
     a->d[i] |= (((BN_ULONG)1) << j);
@@ -828,8 +833,9 @@ int BN_security_bits(int L, int N)
 
 void BN_zero_ex(BIGNUM *a)
 {
-    a->top = 0;
     a->neg = 0;
+    a->top = 0;
+    a->flags &= ~BN_FLG_FIXED_TOP;
 }
 
 int BN_abs_is_word(const BIGNUM *a, const BN_ULONG w)
@@ -953,5 +959,6 @@ void bn_correct_top(BIGNUM *a)
     }
     if (a->top == 0)
         a->neg = 0;
+    a->flags &= ~BN_FLG_FIXED_TOP;
     bn_pollute(a);
 }