ec key validation checks updated

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8564)

1 files changed, 55 insertions, 2 deletions

diff --git a/test/ectest.c b/test/ectest.c
index 7236b43bd1..ab75acea1b 100644
--- a/test/ectest.c
+++ b/test/ectest.c
@@ -1855,7 +1855,59 @@ err:
     OPENSSL_free(buf);
     return r;
 }
-#endif
+
+static int check_ec_key_field_public_range_test(int id)
+{
+    int ret = 0, type = 0;
+    const EC_POINT *pub = NULL;
+    const EC_GROUP *group = NULL;
+    const EC_METHOD *meth = NULL;
+    const BIGNUM *field = NULL;
+    BIGNUM *x = NULL, *y = NULL;
+    EC_KEY *key = NULL;
+
+    if (!(TEST_ptr(x = BN_new())
+          && TEST_ptr(y = BN_new())
+          && TEST_ptr(key = EC_KEY_new_by_curve_name(curves[id].nid))
+          && TEST_ptr(group = EC_KEY_get0_group(key))
+          && TEST_ptr(meth = EC_GROUP_method_of(group))
+          && TEST_ptr(field = EC_GROUP_get0_field(group))
+          && TEST_int_gt(EC_KEY_generate_key(key), 0)
+          && TEST_int_gt(EC_KEY_check_key(key), 0)
+          && TEST_ptr(pub = EC_KEY_get0_public_key(key))
+          && TEST_int_gt(EC_POINT_get_affine_coordinates(group, pub, x, y,
+                                                         NULL), 0)))
+        goto err;
+
+    /*
+     * Make the public point out of range by adding the field (which will still
+     * be the same point on the curve). The add is different for char2 fields.
+     */
+    type = EC_METHOD_get_field_type(meth);
+    if (type == NID_X9_62_characteristic_two_field) {
+        /* test for binary curves */
+        if (!TEST_true(BN_GF2m_add(x, x, field)))
+            goto err;
+    } else if (type == NID_X9_62_prime_field) {
+        /* test for prime curves */
+        if (!TEST_true(BN_add(x, x, field)))
+            goto err;
+    } else {
+        /* this should never happen */
+        TEST_error("Unsupported EC_METHOD field_type");
+        goto err;
+    }
+    if (!TEST_int_le(EC_KEY_set_public_key_affine_coordinates(key, x, y), 0))
+        goto err;
+
+    ret = 1;
+err:
+    BN_free(x);
+    BN_free(y);
+    EC_KEY_free(key);
+    return ret;
+}
+#endif /* OPENSSL_NO_EC */
 
 int setup_tests(void)
 {
@@ -1880,7 +1932,8 @@ int setup_tests(void)
     ADD_TEST(group_field_test);
     ADD_ALL_TESTS(check_named_curve_test, crv_len);
     ADD_ALL_TESTS(check_named_curve_lookup_test, crv_len);
-#endif
+    ADD_ALL_TESTS(check_ec_key_field_public_range_test, crv_len);
+#endif /* OPENSSL_NO_EC */
     return 1;
 }