EC_POINT_is_on_curve does not return a boolean

The function EC_POINT_is_on_curve does not return a boolean value.
It returns 1 if the point is on the curve, 0 if it is not, and -1
on error. Many usages within OpenSSL were incorrectly using this
function and therefore not correctly handling error conditions.

With thanks to the Open Crypto Audit Project for reporting this issue.

Reviewed-by: Kurt Roeckx <kurt@openssl.org>

1 files changed, 12 insertions, 12 deletions

diff --git a/test/ectest.c b/test/ectest.c
index 977b8d969b..c8893f9f23 100644
--- a/test/ectest.c
+++ b/test/ectest.c
@@ -319,7 +319,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, Q, x, 1, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, Q, ctx)) {
+    if (EC_POINT_is_on_curve(group, Q, ctx) <= 0) {
         if (!EC_POINT_get_affine_coordinates_GFp(group, Q, x, y, ctx))
             ABORT;
         fprintf(stderr, "Point is not on curve: x = 0x");
@@ -439,7 +439,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_affine_coordinates_GFp(group, P, x, y, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn(&z, "0100000000000000000001F4C8F927AED3CA752257"))
         ABORT;
@@ -488,7 +488,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, P, x, 1, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn(&z, "FFFFFFFFFFFFFFFFFFFFFFFF99DEF836146BC9B1B4D22831"))
         ABORT;
@@ -541,7 +541,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, P, x, 0, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn
         (&z, "FFFFFFFFFFFFFFFFFFFFFFFFFFFF16A2E0B8F03E13DD29455C5C2A3D"))
@@ -600,7 +600,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, P, x, 1, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn(&z, "FFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E"
                    "84F3B9CAC2FC632551"))
@@ -656,7 +656,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, P, x, 1, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn(&z, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
                    "FFC7634D81F4372DDF581A0DB248B0A77AECEC196ACCC52973"))
@@ -715,7 +715,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_set_compressed_coordinates_GFp(group, P, x, 0, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!BN_hex2bn(&z, "1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
                    "FFFFFFFFFFFFFFFFFFFFA51868783BF2F966B7FCC0148F709A5D03BB5"
@@ -759,7 +759,7 @@ static void prime_field_tests(void)
         ABORT;
     if (!EC_POINT_dbl(group, P, P, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!EC_POINT_invert(group, Q, ctx))
         ABORT;                  /* P = -2Q */
@@ -877,7 +877,7 @@ static void prime_field_tests(void)
 #  define CHAR2_CURVE_TEST_INTERNAL(_name, _p, _a, _b, _x, _y, _y_bit, _order, _cof, _degree, _variable) \
         if (!BN_hex2bn(&x, _x)) ABORT; \
         if (!EC_POINT_set_compressed_coordinates_GF2m(group, P, x, _y_bit, ctx)) ABORT; \
-        if (!EC_POINT_is_on_curve(group, P, ctx)) ABORT; \
+        if (EC_POINT_is_on_curve(group, P, ctx) <= 0) ABORT; \
         if (!BN_hex2bn(&z, _order)) ABORT; \
         if (!BN_hex2bn(&cof, _cof)) ABORT; \
         if (!EC_GROUP_set_generator(group, P, z, cof)) ABORT; \
@@ -895,7 +895,7 @@ static void prime_field_tests(void)
         if (!BN_hex2bn(&x, _x)) ABORT; \
         if (!BN_hex2bn(&y, _y)) ABORT; \
         if (!EC_POINT_set_affine_coordinates_GF2m(group, P, x, y, ctx)) ABORT; \
-        if (!EC_POINT_is_on_curve(group, P, ctx)) ABORT; \
+        if (EC_POINT_is_on_curve(group, P, ctx) <= 0) ABORT; \
         if (!BN_hex2bn(&z, _order)) ABORT; \
         if (!BN_hex2bn(&cof, _cof)) ABORT; \
         if (!EC_GROUP_set_generator(group, P, z, cof)) ABORT; \
@@ -1024,7 +1024,7 @@ static void char2_field_tests(void)
     if (!EC_POINT_set_affine_coordinates_GF2m(group, Q, x, y, ctx))
         ABORT;
 #  endif
-    if (!EC_POINT_is_on_curve(group, Q, ctx)) {
+    if (EC_POINT_is_on_curve(group, Q, ctx) <= 0) {
 /* Change test based on whether binary point compression is enabled or not. */
 #  ifdef OPENSSL_EC_BIN_PT_COMP
         if (!EC_POINT_get_affine_coordinates_GF2m(group, Q, x, y, ctx))
@@ -1245,7 +1245,7 @@ static void char2_field_tests(void)
         ABORT;
     if (!EC_POINT_dbl(group, P, P, ctx))
         ABORT;
-    if (!EC_POINT_is_on_curve(group, P, ctx))
+    if (EC_POINT_is_on_curve(group, P, ctx) <= 0)
         ABORT;
     if (!EC_POINT_invert(group, Q, ctx))
         ABORT;                  /* P = -2Q */