Handle evp_tests assumption of EVP_PKEY_FLAG_AUTOARGLEN

Without actually using EVP_PKEY_FLAG_AUTOARGLEN

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4793)

5 files changed, 43 insertions, 8 deletions

diff --git a/crypto/ec/ec_pmeth.c b/crypto/ec/ec_pmeth.c
index 5f3f56c961..08dda12eeb 100644
--- a/crypto/ec/ec_pmeth.c
+++ b/crypto/ec/ec_pmeth.c
@@ -221,8 +221,14 @@ static int pkey_ecies_encrypt(EVP_PKEY_CTX *ctx,
 # if defined(OPENSSL_NO_SM2)
         ret = -1;
 # else
-        ret = SM2_encrypt(ec, EVP_get_digestbynid(md_type),
-                          in, inlen, out, outlen);
+        if (out == NULL) {
+            *outlen = SM2_ciphertext_size(ec, EVP_get_digestbynid(md_type), inlen);
+            ret = 1;
+        }
+        else {
+            ret = SM2_encrypt(ec, EVP_get_digestbynid(md_type),
+                              in, inlen, out, outlen);
+        }
 # endif
     } else {
         /* standard ECIES not implemented */
@@ -252,8 +258,14 @@ static int pkey_ecies_decrypt(EVP_PKEY_CTX *ctx,
 # if defined(OPENSSL_NO_SM2)
         ret = -1;
 # else
-        ret = SM2_decrypt(ec, EVP_get_digestbynid(md_type),
-                          in, inlen, out, outlen);
+        if (out == NULL) {
+            *outlen = SM2_plaintext_size(ec, EVP_get_digestbynid(md_type), inlen);
+            ret = 1;
+        }
+        else {
+            ret = SM2_decrypt(ec, EVP_get_digestbynid(md_type),
+                              in, inlen, out, outlen);
+        }
 # endif
     } else {
         /* standard ECIES not implemented */
@@ -510,7 +522,7 @@ static int pkey_ec_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 
 const EVP_PKEY_METHOD ec_pkey_meth = {
     EVP_PKEY_EC,
-    EVP_PKEY_FLAG_AUTOARGLEN,
+    0,
     pkey_ec_init,
     pkey_ec_copy,
     pkey_ec_cleanup,
diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index 7e7be9d04a..b308e5b2e5 100644
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -57,10 +57,23 @@ static size_t EC_field_size(const EC_GROUP *group)
     return field_size;
 }
 
+size_t SM2_plaintext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len)
+{
+    const size_t field_size = EC_field_size(EC_KEY_get0_group(key));
+    const size_t md_size = EVP_MD_size(digest);
+
+    const size_t overhead = 10 + 2 * field_size + md_size;
+    if(msg_len <= overhead)
+       return 0;
+
+    return msg_len - overhead;
+}
+
 size_t SM2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len)
 {
-    return 10 + 2 * EC_field_size(EC_KEY_get0_group(key)) +
-        EVP_MD_size(digest) + msg_len;
+    const size_t field_size = EC_field_size(EC_KEY_get0_group(key));
+    const size_t md_size = EVP_MD_size(digest);
+    return 10 + 2 * field_size + md_size + msg_len;
 }
 
 int SM2_encrypt(const EC_KEY *key,
diff --git a/include/openssl/sm2.h b/include/openssl/sm2.h
index d12dcadef2..892ffb152a 100644
--- a/include/openssl/sm2.h
+++ b/include/openssl/sm2.h
@@ -54,6 +54,10 @@ size_t SM2_ciphertext_size(const EC_KEY *key,
                            const EVP_MD *digest,
                            size_t msg_len);
 
+size_t SM2_plaintext_size(const EC_KEY *key,
+                          const EVP_MD *digest,
+                          size_t msg_len);
+
 int SM2_encrypt(const EC_KEY *key,
                 const EVP_MD *digest,
                 const uint8_t *msg,
diff --git a/test/sm2crypttest.c b/test/sm2crypttest.c
index e1e00d8aa4..961256bc90 100644
--- a/test/sm2crypttest.c
+++ b/test/sm2crypttest.c
@@ -139,6 +139,7 @@ static int test_sm2(const EC_GROUP *group,
     unsigned char *expected = OPENSSL_hexstr2buf(ctext_hex, NULL);
 
     size_t ctext_len = 0;
+    size_t ptext_len = 0;
     uint8_t *ctext = NULL;
     uint8_t *recovered = NULL;
     size_t recovered_len = msg_len;
@@ -172,7 +173,11 @@ static int test_sm2(const EC_GROUP *group,
     if (rc == 0)
         goto done;
 
-    recovered = OPENSSL_zalloc(msg_len);
+    ptext_len = SM2_plaintext_size(key, digest, ctext_len);
+
+    TEST_int_eq(ptext_len, msg_len);
+
+    recovered = OPENSSL_zalloc(ptext_len);
     if (recovered == NULL)
         goto done;
     rc = SM2_decrypt(key, digest, ctext, ctext_len, recovered, &recovered_len);
diff --git a/util/libcrypto.num b/util/libcrypto.num
index 0dfb0cd661..96cbb2c3df 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -4523,3 +4523,4 @@ SM2_verify                              4464	1_1_1	EXIST::FUNCTION:
 SM2_do_verify                           4465	1_1_1	EXIST::FUNCTION:
 SM2_sign                                4466	1_1_1	EXIST::FUNCTION:
 ERR_load_SM2_strings                    4467	1_1_1	EXIST::FUNCTION:
+SM2_plaintext_size                      4468	1_1_1	EXIST::FUNCTION: