Disallow Ed448 signature malleability

Check that s is less than the order before attempting to verify the
signature as per RFC8032 5.2.7

Fixes #7706

Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
(Merged from https://github.com/openssl/openssl/pull/7748)

(cherry picked from commit 08afd2f37a4465c90b9b9e2081c9e8df4726db89)

1 files changed, 28 insertions, 2 deletions

diff --git a/crypto/ec/curve448/eddsa.c b/crypto/ec/curve448/eddsa.c
index 909413a535..b28f7dff91 100644
--- a/crypto/ec/curve448/eddsa.c
+++ b/crypto/ec/curve448/eddsa.c
@@ -246,10 +246,36 @@ c448_error_t c448_ed448_verify(
                     uint8_t context_len)
 {
     curve448_point_t pk_point, r_point;
-    c448_error_t error =
-        curve448_point_decode_like_eddsa_and_mul_by_ratio(pk_point, pubkey);
+    c448_error_t error;
     curve448_scalar_t challenge_scalar;
     curve448_scalar_t response_scalar;
+    /* Order in little endian format */
+    static const uint8_t order[] = {
+        0xF3, 0x44, 0x58, 0xAB, 0x92, 0xC2, 0x78, 0x23, 0x55, 0x8F, 0xC5, 0x8D,
+        0x72, 0xC2, 0x6C, 0x21, 0x90, 0x36, 0xD6, 0xAE, 0x49, 0xDB, 0x4E, 0xC4,
+        0xE9, 0x23, 0xCA, 0x7C, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x3F, 0x00
+    };
+    int i;
+
+    /*
+     * Check that s (second 57 bytes of the sig) is less than the order. Both
+     * s and the order are in little-endian format. This can be done in
+     * variable time, since if this is not the case the signature if publicly
+     * invalid.
+     */
+    for (i = EDDSA_448_PUBLIC_BYTES - 1; i >= 0; i--) {
+        if (signature[i + EDDSA_448_PUBLIC_BYTES] > order[i])
+            return C448_FAILURE;
+        if (signature[i + EDDSA_448_PUBLIC_BYTES] < order[i])
+            break;
+    }
+    if (i < 0)
+        return C448_FAILURE;
+
+    error =
+        curve448_point_decode_like_eddsa_and_mul_by_ratio(pk_point, pubkey);
 
     if (C448_SUCCESS != error)
         return error;