Improve use of the test framework in the SM2 internal tests

Also general clean up of those tests

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6386)

3 files changed, 43 insertions, 23 deletions

diff --git a/crypto/ec/ec_pmeth.c b/crypto/ec/ec_pmeth.c
index eefe2d0cd5..50331d32c4 100644
--- a/crypto/ec/ec_pmeth.c
+++ b/crypto/ec/ec_pmeth.c
@@ -223,9 +223,11 @@ static int pkey_ecies_encrypt(EVP_PKEY_CTX *ctx,
             md_type = NID_sm3;
 
         if (out == NULL) {
-            *outlen = SM2_ciphertext_size(ec, EVP_get_digestbynid(md_type),
-                                          inlen);
-            ret = 1;
+            if (!SM2_ciphertext_size(ec, EVP_get_digestbynid(md_type), inlen,
+                                     outlen))
+                ret = -1;
+            else
+                ret = 1;
         }
         else {
             ret = SM2_encrypt(ec, EVP_get_digestbynid(md_type),
@@ -261,9 +263,11 @@ static int pkey_ecies_decrypt(EVP_PKEY_CTX *ctx,
             md_type = NID_sm3;
 
         if (out == NULL) {
-            *outlen = SM2_plaintext_size(ec, EVP_get_digestbynid(md_type),
-                                         inlen);
-            ret = 1;
+            if (!SM2_plaintext_size(ec, EVP_get_digestbynid(md_type), inlen,
+                                    outlen))
+                ret = -1;
+            else
+                ret = 1;
         }
         else {
             ret = SM2_decrypt(ec, EVP_get_digestbynid(md_type),
diff --git a/crypto/include/internal/sm2.h b/crypto/include/internal/sm2.h
index af24c0146b..91420a520c 100644
--- a/crypto/include/internal/sm2.h
+++ b/crypto/include/internal/sm2.h
@@ -57,13 +57,11 @@ int SM2_verify(int type, const unsigned char *dgst, int dgstlen,
 /*
  * SM2 encryption
  */
-size_t SM2_ciphertext_size(const EC_KEY *key,
-                           const EVP_MD *digest,
-                           size_t msg_len);
+int SM2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len,
+                        size_t *ct_size);
 
-size_t SM2_plaintext_size(const EC_KEY *key,
-                          const EVP_MD *digest,
-                          size_t msg_len);
+int SM2_plaintext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len,
+                       size_t *pt_size);
 
 int SM2_encrypt(const EC_KEY *key,
                 const EVP_MD *digest,
@@ -76,8 +74,6 @@ int SM2_decrypt(const EC_KEY *key,
                 const uint8_t *ciphertext,
                 size_t ciphertext_len, uint8_t *ptext_buf, size_t *ptext_len);
 
-int ERR_load_SM2_strings(void);
-
 #  ifdef __cplusplus
 }
 #  endif
diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index ab2d18ee2e..2a0fdceea7 100644
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -59,23 +59,43 @@ static size_t EC_field_size(const EC_GROUP *group)
     return field_size;
 }
 
-size_t SM2_plaintext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len)
+int SM2_plaintext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len,
+                       size_t *pt_size)
 {
     const size_t field_size = EC_field_size(EC_KEY_get0_group(key));
-    const size_t md_size = EVP_MD_size(digest);
+    const int md_size = EVP_MD_size(digest);
+    size_t overhead;
 
-    const size_t overhead = 10 + 2 * field_size + md_size;
-    if(msg_len <= overhead)
-       return 0;
+    if (md_size < 0) {
+        SM2err(SM2_F_SM2_ENCRYPT, SM2_R_INVALID_DIGEST);
+        return 0;
+    }
+    if (field_size == 0) {
+        SM2err(SM2_F_SM2_ENCRYPT, SM2_R_INVALID_FIELD);
+        return 0;
+    }
+
+    overhead = 10 + 2 * field_size + (size_t)md_size;
+    if(msg_len <= overhead) {
+        SM2err(SM2_F_SM2_ENCRYPT, SM2_R_INVALID_ENCODING);
+        return 0;
+    }
 
-    return msg_len - overhead;
+    *pt_size = msg_len - overhead;
+    return 1;
 }
 
-size_t SM2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len)
+int SM2_ciphertext_size(const EC_KEY *key, const EVP_MD *digest, size_t msg_len,
+                        size_t *ct_size)
 {
     const size_t field_size = EC_field_size(EC_KEY_get0_group(key));
-    const size_t md_size = EVP_MD_size(digest);
-    return 10 + 2 * field_size + md_size + msg_len;
+    const int md_size = EVP_MD_size(digest);
+
+    if (field_size == 0 || md_size < 0)
+        return 0;
+
+    *ct_size = 10 + 2 * field_size + (size_t)md_size + msg_len;
+    return 1;
 }
 
 int SM2_encrypt(const EC_KEY *key,