ec/ecp_nistp*.c: sanitize for undefined/implmentation-specific behaviour.

Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
(Merged from https://github.com/openssl/openssl/pull/4974)

(cherry picked from commit 8af7e94d7b720224547efa7e3364857f7f666dd4)

3 files changed, 19 insertions, 20 deletions

diff --git a/crypto/ec/ecp_nistp224.c b/crypto/ec/ecp_nistp224.c
index 1ff1344da6..0cd994fc23 100644
--- a/crypto/ec/ecp_nistp224.c
+++ b/crypto/ec/ecp_nistp224.c
@@ -50,7 +50,6 @@ typedef __uint128_t uint128_t;  /* nonstandard; implemented by gcc on 64-bit
 
 typedef uint8_t u8;
 typedef uint64_t u64;
-typedef int64_t s64;
 
 /******************************************************************************/
 /*-
diff --git a/crypto/ec/ecp_nistp256.c b/crypto/ec/ecp_nistp256.c
index 119be01be0..ffd2a7d93a 100644
--- a/crypto/ec/ecp_nistp256.c
+++ b/crypto/ec/ecp_nistp256.c
@@ -53,7 +53,6 @@ typedef __int128_t int128_t;
 typedef uint8_t u8;
 typedef uint32_t u32;
 typedef uint64_t u64;
-typedef int64_t s64;
 
 /*
  * The underlying field. P256 operates over GF(2^256-2^224+2^192+2^96-1). We
@@ -394,7 +393,7 @@ static void felem_shrink(smallfelem out, const felem in)
 {
     felem tmp;
     u64 a, b, mask;
-    s64 high, low;
+    u64 high, low;
     static const u64 kPrime3Test = 0x7fffffff00000001ul; /* 2^63 - 2^32 + 1 */
 
     /* Carry 2->3 */
@@ -435,29 +434,31 @@ static void felem_shrink(smallfelem out, const felem in)
      * In order to make space in tmp[3] for the carry from 2 -> 3, we
      * conditionally subtract kPrime if tmp[3] is large enough.
      */
-    high = tmp[3] >> 64;
+    high = (u64)(tmp[3] >> 64);
     /* As tmp[3] < 2^65, high is either 1 or 0 */
-    high <<= 63;
-    high >>= 63;
+    high = 0 - high;
     /*-
      * high is:
      *   all ones   if the high word of tmp[3] is 1
-     *   all zeros  if the high word of tmp[3] if 0 */
-    low = tmp[3];
-    mask = low >> 63;
+     *   all zeros  if the high word of tmp[3] if 0
+     */
+    low = (u64)tmp[3];
+    mask = 0 - (low >> 63);
     /*-
      * mask is:
      *   all ones   if the MSB of low is 1
-     *   all zeros  if the MSB of low if 0 */
+     *   all zeros  if the MSB of low if 0
+     */
     low &= bottom63bits;
     low -= kPrime3Test;
     /* if low was greater than kPrime3Test then the MSB is zero */
     low = ~low;
-    low >>= 63;
+    low = 0 - (low >> 63);
     /*-
      * low is:
      *   all ones   if low was > kPrime3Test
-     *   all zeros  if low was <= kPrime3Test */
+     *   all zeros  if low was <= kPrime3Test
+     */
     mask = (mask & low) | high;
     tmp[0] -= mask & kPrime[0];
     tmp[1] -= mask & kPrime[1];
@@ -891,7 +892,7 @@ static void felem_contract(smallfelem out, const felem in)
         equal &= equal << 4;
         equal &= equal << 2;
         equal &= equal << 1;
-        equal = ((s64) equal) >> 63;
+        equal = 0 - (equal >> 63);
 
         all_equal_so_far &= equal;
     }
@@ -958,7 +959,7 @@ static limb smallfelem_is_zero(const smallfelem small)
     is_zero &= is_zero << 4;
     is_zero &= is_zero << 2;
     is_zero &= is_zero << 1;
-    is_zero = ((s64) is_zero) >> 63;
+    is_zero = 0 - (is_zero >> 63);
 
     is_p = (small[0] ^ kPrime[0]) |
         (small[1] ^ kPrime[1]) |
@@ -970,7 +971,7 @@ static limb smallfelem_is_zero(const smallfelem small)
     is_p &= is_p << 4;
     is_p &= is_p << 2;
     is_p &= is_p << 1;
-    is_p = ((s64) is_p) >> 63;
+    is_p = 0 - (is_p >> 63);
 
     is_zero |= is_p;
 
diff --git a/crypto/ec/ecp_nistp521.c b/crypto/ec/ecp_nistp521.c
index def8f9328b..133f089fd2 100644
--- a/crypto/ec/ecp_nistp521.c
+++ b/crypto/ec/ecp_nistp521.c
@@ -50,7 +50,6 @@ typedef __uint128_t uint128_t;  /* nonstandard; implemented by gcc on 64-bit
 
 typedef uint8_t u8;
 typedef uint64_t u64;
-typedef int64_t s64;
 
 /*
  * The underlying field. P521 operates over GF(2^521-1). We can serialise an
@@ -867,7 +866,7 @@ static limb felem_is_zero(const felem in)
      * We know that ftmp[i] < 2^63, therefore the only way that the top bit
      * can be set is if is_zero was 0 before the decrement.
      */
-    is_zero = ((s64) is_zero) >> 63;
+    is_zero = 0 - (is_zero >> 63);
 
     is_p = ftmp[0] ^ kPrime[0];
     is_p |= ftmp[1] ^ kPrime[1];
@@ -880,7 +879,7 @@ static limb felem_is_zero(const felem in)
     is_p |= ftmp[8] ^ kPrime[8];
 
     is_p--;
-    is_p = ((s64) is_p) >> 63;
+    is_p = 0 - (is_p >> 63);
 
     is_zero |= is_p;
     return is_zero;
@@ -951,7 +950,7 @@ static void felem_contract(felem out, const felem in)
     is_p &= is_p << 4;
     is_p &= is_p << 2;
     is_p &= is_p << 1;
-    is_p = ((s64) is_p) >> 63;
+    is_p = 0 - (is_p >> 63);
     is_p = ~is_p;
 
     /* is_p is 0 iff |out| == 2^521-1 and all ones otherwise */
@@ -977,7 +976,7 @@ static void felem_contract(felem out, const felem in)
     is_greater |= is_greater << 4;
     is_greater |= is_greater << 2;
     is_greater |= is_greater << 1;
-    is_greater = ((s64) is_greater) >> 63;
+    is_greater = 0 - (is_greater >> 63);
 
     out[0] -= kPrime[0] & is_greater;
     out[1] -= kPrime[1] & is_greater;