Fix KMAC bounds checks.

Setting an output length higher than 8191 was causing a buffer overflow.
This was reported by Acumen (FIPS lab).

The max output size has increased to ~2M and it now checks this during set_parameters.

The encoder related functions now pass in the maximum size of the output buffer so they
can correctly check their size. kmac_bytepad_encode_key() calls bytepad twice in
order to calculate and check the length before encoding.

Note that right_encode() is currently only used in one place but this
may change if other algorithms are supported (such as TupleHash).

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15106)

4 files changed, 56 insertions, 39 deletions

diff --git a/providers/common/provider_err.c b/providers/common/provider_err.c
index 8b5d0008f9..eff523b579 100644
--- a/providers/common/provider_err.c
+++ b/providers/common/provider_err.c
@@ -89,6 +89,8 @@ static const ERR_STRING_DATA PROV_str_reasons[] = {
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MAC), "invalid mac"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MGF1_MD), "invalid mgf1 md"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MODE), "invalid mode"},
+    {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_OUTPUT_LENGTH),
+    "invalid output length"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_PADDING_MODE),
     "invalid padding mode"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_PUBINFO), "invalid pubinfo"},
@@ -112,7 +114,7 @@ static const ERR_STRING_DATA PROV_str_reasons[] = {
     "key size too small"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_LENGTH_TOO_LARGE), "length too large"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_MISMATCHING_DOMAIN_PARAMETERS),
-    "mismatching shared parameters"},
+    "mismatching domain parameters"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_MISSING_CEK_ALG), "missing cek alg"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_MISSING_CIPHER), "missing cipher"},
     {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_MISSING_CONFIG_DATA),
diff --git a/providers/fips-sources.checksums b/providers/fips-sources.checksums
index 8c46849215..a7ee231b15 100644
--- a/providers/fips-sources.checksums
+++ b/providers/fips-sources.checksums
@@ -328,7 +328,7 @@ f3b089fd3dcccc8e3ebfbbdbf87c47d58330f82bd0e2a1223da74977930cccf1  providers/comm
 390b2b6ba321bddc416688d4a51d9e04db7d84d4f398947d496d043e8fb22a01  providers/common/der/der_sm2_sig.c
 d447cd774869da68a2cc0bbb19c547ee6ed4858c7aee1f3d5bba7796f97823a9  providers/common/digest_to_nid.c
 737cc1228106e555e9bab24e3c2438982e04e05b0d5b9ee6995d71df16c49143  providers/common/provider_ctx.c
-fcbb0f2859f28ea1eb3922447bb96588d2097695f9ce23c3c64025bfbe9d2bad  providers/common/provider_err.c
+71c3fbb9bd80f5e7a217cf8005df61f96a645fbdd9daca9949ceef6d33a1feb0  providers/common/provider_err.c
 9eae3e2cac89c7b63d091fdca1b6d80c5c5d52aa79c8ba4ce0158c5437ad62f3  providers/common/provider_seeding.c
 eec462d685dd3b4764b076a3c18ecd9dd254350a0b78ddc2f8a60587829e1ce3  providers/common/provider_util.c
 494723d55bc6ecdb70f59499a2c42260cabc5fa30681ac3b48267dfa242158b3  providers/common/securitycheck.c
@@ -432,7 +432,7 @@ c48eb00f0de1c28baaa3cf7c0e85d4d2a20592783aa545f8934da487c05a3e87  providers/impl
 25d20ceb61cadb495ec890ae2c49c5c1c840b39ac77f20058ee87249cab341ef  providers/implementations/macs/cmac_prov.c
 f51b074d55028d3e24656da348d21ca79f6680fdb30383d936251f1b3467caab  providers/implementations/macs/gmac_prov.c
 35505704fda658c0911f95974913c1f2dd75c8f91c5d2ec597c70c52624bdfdf  providers/implementations/macs/hmac_prov.c
-3201d82d1e17c22a80b26dedae627be10b6dc1af623d1fd0c3c923e0125a42e7  providers/implementations/macs/kmac_prov.c
+e42823cce1d08d9cb6cb32cc6b913241573c2cbbd856ff77a331b0956ee5aa02  providers/implementations/macs/kmac_prov.c
 94d80682125b40ba694242fdfa978b802c6e70f2b0167215c9d689c0ccf5820f  providers/implementations/macs/poly1305_prov.c
 d594704aa3173afdb2b1e95253285cdb245a42078f9ca06b68aaeecb858b10fd  providers/implementations/macs/siphash_prov.c
 dcc1afbe2965de7c5ac0a17ab1b19b8ed512049376833cb410db30f8dc4e2064  providers/implementations/rands/crngt.c
diff --git a/providers/fips.checksum b/providers/fips.checksum
index 468c3c986e..ff7a1c2c78 100644
--- a/providers/fips.checksum
+++ b/providers/fips.checksum
@@ -1 +1 @@
-16e17331a77aed06b6537cafdacd35df08fbc888c04eb7cca928a4a39d858642  providers/fips-sources.checksums
+b998b19b940b606688e4711014407c48c3fca4c58b2fdc60ac64c1cef94861c1  providers/fips-sources.checksums
diff --git a/providers/implementations/macs/kmac_prov.c b/providers/implementations/macs/kmac_prov.c
index 111e0e8ba7..c95cf57ffb 100644
--- a/providers/implementations/macs/kmac_prov.c
+++ b/providers/implementations/macs/kmac_prov.c
@@ -78,10 +78,14 @@ static OSSL_FUNC_mac_init_fn kmac_init;
 static OSSL_FUNC_mac_update_fn kmac_update;
 static OSSL_FUNC_mac_final_fn kmac_final;
 
-#define KMAC_MAX_BLOCKSIZE ((1600 - 128*2) / 8) /* 168 */
+#define KMAC_MAX_BLOCKSIZE ((1600 - 128 * 2) / 8) /* 168 */
 
-/* Length encoding will be  a 1 byte size + length in bits (2 bytes max) */
-#define KMAC_MAX_ENCODED_HEADER_LEN 3
+/*
+ * Length encoding will be  a 1 byte size + length in bits (3 bytes max)
+ * This gives a range of 0..0XFFFFFF bits = 2097151 bytes).
+ */
+#define KMAC_MAX_OUTPUT_LEN (0xFFFFFF / 8)
+#define KMAC_MAX_ENCODED_HEADER_LEN (1 + 3)
 
 /*
  * Restrict the maximum length of the customisation string.  This must not
@@ -92,12 +96,13 @@ static OSSL_FUNC_mac_final_fn kmac_final;
 /* Maximum size of encoded custom string */
 #define KMAC_MAX_CUSTOM_ENCODED (KMAC_MAX_CUSTOM + KMAC_MAX_ENCODED_HEADER_LEN)
 
-/* Maximum key size in bytes = 2040 / 8 */
-#define KMAC_MAX_KEY 255
+/* Maximum key size in bytes = 256 (2048 bits) */
+#define KMAC_MAX_KEY 256
+#define KMAC_MIN_KEY 4
 
 /*
  * Maximum Encoded Key size will be padded to a multiple of the blocksize
- * i.e KMAC_MAX_KEY + KMAC_MAX_ENCODED_LEN = 258
+ * i.e KMAC_MAX_KEY + KMAC_MAX_ENCODED_HEADER_LEN = 256 + 4
  * Padded to a multiple of KMAC_MAX_BLOCKSIZE
  */
 #define KMAC_MAX_KEY_ENCODED (KMAC_MAX_BLOCKSIZE * 2)
@@ -107,7 +112,6 @@ static const unsigned char kmac_string[] = {
     0x01, 0x20, 0x4B, 0x4D, 0x41, 0x43
 };
 
-
 #define KMAC_FLAG_XOF_MODE          1
 
 struct kmac_data_st {
@@ -124,14 +128,16 @@ struct kmac_data_st {
     unsigned char custom[KMAC_MAX_CUSTOM_ENCODED];
 };
 
-static int encode_string(unsigned char *out, size_t *out_len,
+static int encode_string(unsigned char *out, size_t out_max_len, size_t *out_len,
                          const unsigned char *in, size_t in_len);
-static int right_encode(unsigned char *out, size_t *out_len, size_t bits);
+static int right_encode(unsigned char *out, size_t out_max_len, size_t *out_len,
+                        size_t bits);
 static int bytepad(unsigned char *out, size_t *out_len,
                    const unsigned char *in1, size_t in1_len,
                    const unsigned char *in2, size_t in2_len,
                    size_t w);
-static int kmac_bytepad_encode_key(unsigned char *out, size_t *out_len,
+static int kmac_bytepad_encode_key(unsigned char *out, size_t out_max_len,
+                                   size_t *out_len,
                                    const unsigned char *in, size_t in_len,
                                    size_t w);
 
@@ -246,7 +252,7 @@ static int kmac_setkey(struct kmac_data_st *kctx, const unsigned char *key,
     const EVP_MD *digest = ossl_prov_digest_md(&kctx->digest);
     int w = EVP_MD_block_size(digest);
 
-    if (keylen < 4 || keylen > KMAC_MAX_KEY) {
+    if (keylen < KMAC_MIN_KEY || keylen > KMAC_MAX_KEY) {
         ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH);
         return 0;
     }
@@ -254,7 +260,7 @@ static int kmac_setkey(struct kmac_data_st *kctx, const unsigned char *key,
         ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH);
         return 0;
     }
-    if (!kmac_bytepad_encode_key(kctx->key, &kctx->key_len,
+    if (!kmac_bytepad_encode_key(kctx->key, sizeof(kctx->key), &kctx->key_len,
                                  key, keylen, (size_t)w))
         return 0;
     return 1;
@@ -346,7 +352,7 @@ static int kmac_final(void *vmacctx, unsigned char *out, size_t *outl,
     /* KMAC XOF mode sets the encoded length to 0 */
     lbits = (kctx->xof_mode ? 0 : (kctx->out_len * 8));
 
-    ok = right_encode(encoded_outlen, &len, lbits)
+    ok = right_encode(encoded_outlen, sizeof(encoded_outlen), &len, lbits)
         && EVP_DigestUpdate(ctx, encoded_outlen, len)
         && EVP_DigestFinalXOF(ctx, out, kctx->out_len);
     *outl = kctx->out_len;
@@ -406,9 +412,17 @@ static int kmac_set_ctx_params(void *vmacctx, const OSSL_PARAM *params)
     if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_XOF)) != NULL
         && !OSSL_PARAM_get_int(p, &kctx->xof_mode))
         return 0;
-    if (((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_SIZE)) != NULL)
-        && !OSSL_PARAM_get_size_t(p, &kctx->out_len))
-        return 0;
+    if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_SIZE)) != NULL) {
+        size_t sz = 0;
+
+        if (!OSSL_PARAM_get_size_t(p, &sz))
+            return 0;
+        if (sz > KMAC_MAX_OUTPUT_LEN) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_OUTPUT_LENGTH);
+            return 0;
+        }
+        kctx->out_len = sz;
+    }
     if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_KEY)) != NULL
             && !kmac_setkey(kctx, p->data, p->data_size))
         return 0;
@@ -418,16 +432,14 @@ static int kmac_set_ctx_params(void *vmacctx, const OSSL_PARAM *params)
             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CUSTOM_LENGTH);
             return 0;
         }
-        if (!encode_string(kctx->custom, &kctx->custom_len,
+        if (!encode_string(kctx->custom, sizeof(kctx->custom), &kctx->custom_len,
                            p->data, p->data_size))
             return 0;
     }
     return 1;
 }
 
-/*
- * Encoding/Padding Methods.
- */
+/* Encoding/Padding Methods. */
 
 /* Returns the number of bytes required to store 'bits' into a byte array */
 static unsigned int get_encode_size(size_t bits)
@@ -450,15 +462,14 @@ static unsigned int get_encode_size(size_t bits)
  * *out_len.
  *
  * e.g if bits = 32, out[2] = { 0x20, 0x01 }
- *
  */
-static int right_encode(unsigned char *out, size_t *out_len, size_t bits)
+static int right_encode(unsigned char *out, size_t out_max_len, size_t *out_len,
+                        size_t bits)
 {
     unsigned int len = get_encode_size(bits);
     int i;
 
-    /* The length is constrained to a single byte: 2040/8 = 255 */
-    if (len > 0xFF) {
+    if (len >= out_max_len) {
         ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
         return 0;
     }
@@ -483,17 +494,19 @@ static int right_encode(unsigned char *out, size_t *out_len, size_t bits)
  * e.g- in="KMAC" gives out[6] = { 0x01, 0x20, 0x4B, 0x4D, 0x41, 0x43 }
  *                                 len   bits    K     M     A     C
  */
-static int encode_string(unsigned char *out, size_t *out_len,
+static int encode_string(unsigned char *out, size_t out_max_len, size_t *out_len,
                          const unsigned char *in, size_t in_len)
 {
     if (in == NULL) {
         *out_len = 0;
     } else {
-        size_t i, bits, len;
+        size_t i, bits, len, sz;
 
         bits = 8 * in_len;
         len = get_encode_size(bits);
-        if (len > 0xFF) {
+        sz = 1 + len + in_len;
+
+        if (sz > out_max_len) {
             ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
             return 0;
         }
@@ -504,7 +517,7 @@ static int encode_string(unsigned char *out, size_t *out_len,
             bits >>= 8;
         }
         memcpy(out + len + 1, in, in_len);
-        *out_len = (1 + len + in_len);
+        *out_len = sz;
     }
     return 1;
 }
@@ -560,20 +573,22 @@ static int bytepad(unsigned char *out, size_t *out_len,
     return 1;
 }
 
-/*
- * Returns out = bytepad(encode_string(in), w)
- */
-static int kmac_bytepad_encode_key(unsigned char *out, size_t *out_len,
+/* Returns out = bytepad(encode_string(in), w) */
+static int kmac_bytepad_encode_key(unsigned char *out, size_t out_max_len,
+                                   size_t *out_len,
                                    const unsigned char *in, size_t in_len,
                                    size_t w)
 {
     unsigned char tmp[KMAC_MAX_KEY + KMAC_MAX_ENCODED_HEADER_LEN];
     size_t tmp_len;
 
-    if (!encode_string(tmp, &tmp_len, in, in_len))
+    if (!encode_string(tmp, sizeof(tmp), &tmp_len, in, in_len))
         return 0;
-
-    return bytepad(out, out_len, tmp, tmp_len, NULL, 0, w);
+    if (!bytepad(NULL, out_len, tmp, tmp_len, NULL, 0, w))
+        return 0;
+    if (!ossl_assert(*out_len <= out_max_len))
+        return 0;
+    return bytepad(out, NULL, tmp, tmp_len, NULL, 0, w);
 }
 
 const OSSL_DISPATCH ossl_kmac128_functions[] = {