[ec/ecp_nistp*.c] remove flip_endian()

Replace flip_endian() by using the little endian specific
bn_bn2lebinpad() and bn_lebin2bn().

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/9793)

3 files changed, 65 insertions, 87 deletions

diff --git a/crypto/ec/ecp_nistp224.c b/crypto/ec/ecp_nistp224.c
index 877edc9cf0..5df7e745d6 100644
--- a/crypto/ec/ecp_nistp224.c
+++ b/crypto/ec/ecp_nistp224.c
@@ -37,7 +37,7 @@
 # include <string.h>
 # include <openssl/err.h>
 # include "ec_lcl.h"
-# include "bn_int.h" /* bn_bn2binpad */
+# include "bn_int.h" /* bn_bn2lebinpad, bn_lebin2bn */
 
 # if defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
   /* even with gcc, the typedef won't work for 32-bit platforms */
@@ -335,32 +335,21 @@ static void felem_to_bin28(u8 out[28], const felem in)
     }
 }
 
-/* To preserve endianness when using BN_bn2bin and BN_bin2bn */
-static void flip_endian(u8 *out, const u8 *in, unsigned len)
-{
-    unsigned i;
-    for (i = 0; i < len; ++i)
-        out[i] = in[len - 1 - i];
-}
-
 /* From OpenSSL BIGNUM to internal representation */
 static int BN_to_felem(felem out, const BIGNUM *bn)
 {
-    felem_bytearray b_in;
     felem_bytearray b_out;
-    unsigned num_bytes;
+    int num_bytes;
 
-    num_bytes = BN_num_bytes(bn);
-    if (num_bytes > sizeof(b_out)) {
+    if (BN_is_negative(bn)) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    if (BN_is_negative(bn)) {
+    num_bytes = bn_bn2lebinpad(bn, b_out, sizeof(b_out));
+    if (num_bytes < 0) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    num_bytes = bn_bn2binpad(bn, b_in, sizeof(b_in));
-    flip_endian(b_out, b_in, num_bytes);
     bin28_to_felem(out, b_out);
     return 1;
 }
@@ -368,10 +357,9 @@ static int BN_to_felem(felem out, const BIGNUM *bn)
 /* From internal representation to OpenSSL BIGNUM */
 static BIGNUM *felem_to_BN(BIGNUM *out, const felem in)
 {
-    felem_bytearray b_in, b_out;
-    felem_to_bin28(b_in, in);
-    flip_endian(b_out, b_in, sizeof(b_out));
-    return BN_bin2bn(b_out, sizeof(b_out), out);
+    felem_bytearray b_out;
+    felem_to_bin28(b_out, in);
+    return bn_lebin2bn(b_out, sizeof(b_out), out);
 }
 
 /******************************************************************************/
@@ -1425,8 +1413,7 @@ int ec_GFp_nistp224_points_mul(const EC_GROUP *group, EC_POINT *r,
     felem_bytearray *secrets = NULL;
     felem(*pre_comp)[17][3] = NULL;
     felem *tmp_felems = NULL;
-    felem_bytearray tmp;
-    unsigned num_bytes;
+    int num_bytes;
     int have_pre_comp = 0;
     size_t num_points = num;
     felem x_in, y_in, z_in, x_out, y_out, z_out;
@@ -1531,10 +1518,16 @@ int ec_GFp_nistp224_points_mul(const EC_GROUP *group, EC_POINT *r,
                         ECerr(EC_F_EC_GFP_NISTP224_POINTS_MUL, ERR_R_BN_LIB);
                         goto err;
                     }
-                    num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
-                } else
-                    num_bytes = bn_bn2binpad(p_scalar, tmp, sizeof(tmp));
-                flip_endian(secrets[i], tmp, num_bytes);
+                    num_bytes = bn_bn2lebinpad(tmp_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                } else {
+                    num_bytes = bn_bn2lebinpad(p_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                }
+                if (num_bytes < 0) {
+                    ECerr(EC_F_EC_GFP_NISTP224_POINTS_MUL, ERR_R_BN_LIB);
+                    goto err;
+                }
                 /* precompute multiples */
                 if ((!BN_to_felem(x_out, &p->X)) ||
                     (!BN_to_felem(y_out, &p->Y)) ||
@@ -1577,10 +1570,9 @@ int ec_GFp_nistp224_points_mul(const EC_GROUP *group, EC_POINT *r,
                 ECerr(EC_F_EC_GFP_NISTP224_POINTS_MUL, ERR_R_BN_LIB);
                 goto err;
             }
-            num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
+            num_bytes = bn_bn2lebinpad(tmp_scalar, g_secret, sizeof(g_secret));
         } else
-            num_bytes = bn_bn2binpad(scalar, tmp, sizeof(tmp));
-        flip_endian(g_secret, tmp, num_bytes);
+            num_bytes = bn_bn2lebinpad(scalar, g_secret, sizeof(g_secret));
         /* do the multiplication with generator precomputation */
         batch_mul(x_out, y_out, z_out,
                   (const felem_bytearray(*))secrets, num_points,
diff --git a/crypto/ec/ecp_nistp256.c b/crypto/ec/ecp_nistp256.c
index a9b2c062a9..31a0199f88 100644
--- a/crypto/ec/ecp_nistp256.c
+++ b/crypto/ec/ecp_nistp256.c
@@ -38,7 +38,7 @@
 # include <string.h>
 # include <openssl/err.h>
 # include "ec_lcl.h"
-# include "bn_int.h" /* bn_bn2binpad */
+# include "bn_int.h" /* bn_bn2lebinpad, bn_lebin2bn */
 
 # if defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
   /* even with gcc, the typedef won't work for 32-bit platforms */
@@ -145,32 +145,21 @@ static void smallfelem_to_bin32(u8 out[32], const smallfelem in)
     *((u64 *)&out[24]) = in[3];
 }
 
-/* To preserve endianness when using BN_bn2bin and BN_bin2bn */
-static void flip_endian(u8 *out, const u8 *in, unsigned len)
-{
-    unsigned i;
-    for (i = 0; i < len; ++i)
-        out[i] = in[len - 1 - i];
-}
-
 /* BN_to_felem converts an OpenSSL BIGNUM into an felem */
 static int BN_to_felem(felem out, const BIGNUM *bn)
 {
-    felem_bytearray b_in;
     felem_bytearray b_out;
-    unsigned num_bytes;
+    int num_bytes;
 
-    num_bytes = BN_num_bytes(bn);
-    if (num_bytes > sizeof(b_out)) {
+    if (BN_is_negative(bn)) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    if (BN_is_negative(bn)) {
+    num_bytes = bn_bn2lebinpad(bn, b_out, sizeof(b_out));
+    if (num_bytes < 0) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    num_bytes = bn_bn2binpad(bn, b_in, sizeof(b_in));
-    flip_endian(b_out, b_in, num_bytes);
     bin32_to_felem(out, b_out);
     return 1;
 }
@@ -178,10 +167,9 @@ static int BN_to_felem(felem out, const BIGNUM *bn)
 /* felem_to_BN converts an felem into an OpenSSL BIGNUM */
 static BIGNUM *smallfelem_to_BN(BIGNUM *out, const smallfelem in)
 {
-    felem_bytearray b_in, b_out;
-    smallfelem_to_bin32(b_in, in);
-    flip_endian(b_out, b_in, sizeof(b_out));
-    return BN_bin2bn(b_out, sizeof(b_out), out);
+    felem_bytearray b_out;
+    smallfelem_to_bin32(b_out, in);
+    return bn_lebin2bn(b_out, sizeof(b_out), out);
 }
 
 /*-
@@ -2013,8 +2001,8 @@ int ec_GFp_nistp256_points_mul(const EC_GROUP *group, EC_POINT *r,
     felem_bytearray *secrets = NULL;
     smallfelem(*pre_comp)[17][3] = NULL;
     smallfelem *tmp_smallfelems = NULL;
-    felem_bytearray tmp;
-    unsigned i, num_bytes;
+    unsigned i;
+    int num_bytes;
     int have_pre_comp = 0;
     size_t num_points = num;
     smallfelem x_in, y_in, z_in;
@@ -2122,10 +2110,16 @@ int ec_GFp_nistp256_points_mul(const EC_GROUP *group, EC_POINT *r,
                         ECerr(EC_F_EC_GFP_NISTP256_POINTS_MUL, ERR_R_BN_LIB);
                         goto err;
                     }
-                    num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
-                } else
-                    num_bytes = bn_bn2binpad(p_scalar, tmp, sizeof(tmp));
-                flip_endian(secrets[i], tmp, num_bytes);
+                    num_bytes = bn_bn2lebinpad(tmp_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                } else {
+                    num_bytes = bn_bn2lebinpad(p_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                }
+                if (num_bytes < 0) {
+                    ECerr(EC_F_EC_GFP_NISTP256_POINTS_MUL, ERR_R_BN_LIB);
+                    goto err;
+                }
                 /* precompute multiples */
                 if ((!BN_to_felem(x_out, &p->X)) ||
                     (!BN_to_felem(y_out, &p->Y)) ||
@@ -2170,10 +2164,9 @@ int ec_GFp_nistp256_points_mul(const EC_GROUP *group, EC_POINT *r,
                 ECerr(EC_F_EC_GFP_NISTP256_POINTS_MUL, ERR_R_BN_LIB);
                 goto err;
             }
-            num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
+            num_bytes = bn_bn2lebinpad(tmp_scalar, g_secret, sizeof(g_secret));
         } else
-            num_bytes = bn_bn2binpad(scalar, tmp, sizeof(tmp));
-        flip_endian(g_secret, tmp, num_bytes);
+            num_bytes = bn_bn2lebinpad(scalar, g_secret, sizeof(g_secret));
         /* do the multiplication with generator precomputation */
         batch_mul(x_out, y_out, z_out,
                   (const felem_bytearray(*))secrets, num_points,
diff --git a/crypto/ec/ecp_nistp521.c b/crypto/ec/ecp_nistp521.c
index 9765e9866d..4503bb4696 100644
--- a/crypto/ec/ecp_nistp521.c
+++ b/crypto/ec/ecp_nistp521.c
@@ -38,7 +38,7 @@
 # include <string.h>
 # include <openssl/err.h>
 # include "ec_lcl.h"
-# include "bn_int.h" /* bn_bn2binpad */
+# include "bn_int.h" /* bn_bn2lebinpad, bn_lebin2bn */
 
 # if defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
   /* even with gcc, the typedef won't work for 32-bit platforms */
@@ -169,32 +169,21 @@ static void felem_to_bin66(u8 out[66], const felem in)
     (*((limb *) & out[58])) = in[8];
 }
 
-/* To preserve endianness when using BN_bn2bin and BN_bin2bn */
-static void flip_endian(u8 *out, const u8 *in, unsigned len)
-{
-    unsigned i;
-    for (i = 0; i < len; ++i)
-        out[i] = in[len - 1 - i];
-}
-
 /* BN_to_felem converts an OpenSSL BIGNUM into an felem */
 static int BN_to_felem(felem out, const BIGNUM *bn)
 {
-    felem_bytearray b_in;
     felem_bytearray b_out;
-    unsigned num_bytes;
+    int num_bytes;
 
-    num_bytes = BN_num_bytes(bn);
-    if (num_bytes > sizeof(b_out)) {
+    if (BN_is_negative(bn)) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    if (BN_is_negative(bn)) {
+    num_bytes = bn_bn2lebinpad(bn, b_out, sizeof(b_out));
+    if (num_bytes < 0) {
         ECerr(EC_F_BN_TO_FELEM, EC_R_BIGNUM_OUT_OF_RANGE);
         return 0;
     }
-    num_bytes = bn_bn2binpad(bn, b_in, sizeof(b_in));
-    flip_endian(b_out, b_in, num_bytes);
     bin66_to_felem(out, b_out);
     return 1;
 }
@@ -202,10 +191,9 @@ static int BN_to_felem(felem out, const BIGNUM *bn)
 /* felem_to_BN converts an felem into an OpenSSL BIGNUM */
 static BIGNUM *felem_to_BN(BIGNUM *out, const felem in)
 {
-    felem_bytearray b_in, b_out;
-    felem_to_bin66(b_in, in);
-    flip_endian(b_out, b_in, sizeof(b_out));
-    return BN_bin2bn(b_out, sizeof(b_out), out);
+    felem_bytearray b_out;
+    felem_to_bin66(b_out, in);
+    return bn_lebin2bn(b_out, sizeof(b_out), out);
 }
 
 /*-
@@ -1825,8 +1813,8 @@ int ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r,
     felem_bytearray *secrets = NULL;
     felem(*pre_comp)[17][3] = NULL;
     felem *tmp_felems = NULL;
-    felem_bytearray tmp;
-    unsigned i, num_bytes;
+    unsigned i;
+    int num_bytes;
     int have_pre_comp = 0;
     size_t num_points = num;
     felem x_in, y_in, z_in, x_out, y_out, z_out;
@@ -1934,10 +1922,16 @@ int ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r,
                         ECerr(EC_F_EC_GFP_NISTP521_POINTS_MUL, ERR_R_BN_LIB);
                         goto err;
                     }
-                    num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
-                } else
-                    num_bytes = bn_bn2binpad(p_scalar, tmp, sizeof(tmp));
-                flip_endian(secrets[i], tmp, num_bytes);
+                    num_bytes = bn_bn2lebinpad(tmp_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                } else {
+                    num_bytes = bn_bn2lebinpad(p_scalar,
+                                               secrets[i], sizeof(secrets[i]));
+                }
+                if (num_bytes < 0) {
+                    ECerr(EC_F_EC_GFP_NISTP521_POINTS_MUL, ERR_R_BN_LIB);
+                    goto err;
+                }
                 /* precompute multiples */
                 if ((!BN_to_felem(x_out, &p->X)) ||
                     (!BN_to_felem(y_out, &p->Y)) ||
@@ -1980,10 +1974,9 @@ int ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r,
                 ECerr(EC_F_EC_GFP_NISTP521_POINTS_MUL, ERR_R_BN_LIB);
                 goto err;
             }
-            num_bytes = bn_bn2binpad(tmp_scalar, tmp, sizeof(tmp));
+            num_bytes = bn_bn2lebinpad(tmp_scalar, g_secret, sizeof(g_secret));
         } else
-            num_bytes = bn_bn2binpad(scalar, tmp, sizeof(tmp));
-        flip_endian(g_secret, tmp, num_bytes);
+            num_bytes = bn_bn2lebinpad(scalar, g_secret, sizeof(g_secret));
         /* do the multiplication with generator precomputation */
         batch_mul(x_out, y_out, z_out,
                   (const felem_bytearray(*))secrets, num_points,